Merge branch 'master' into staging

15 files changed, 350 insertions, 72 deletions

diff --git a/spec/controllers/compliance_controls_controller_spec.rb b/spec/controllers/compliance_controls_controller_spec.rb
index 34b27530d..61e94025d 100644
--- a/spec/controllers/compliance_controls_controller_spec.rb
+++ b/spec/controllers/compliance_controls_controller_spec.rb
@@ -44,7 +44,7 @@ RSpec.describe ComplianceControlsController, type: :controller do
   describe 'POST #update' do
     it 'should be successful' do
       post :update, compliance_control_set_id: compliance_control_set.id, id: compliance_control.id, compliance_control: compliance_control.as_json.merge(type: 'GenericAttributeControl::MinMax')
-      expect(response).to redirect_to compliance_control_set_compliance_control_path(compliance_control_set, compliance_control)
+      expect(response).to redirect_to compliance_control_set_path(compliance_control_set)
     end
   end
 
diff --git a/spec/factories/imports.rb b/spec/factories/imports.rb
index 2c53106c3..e07447b60 100644
--- a/spec/factories/imports.rb
+++ b/spec/factories/imports.rb
@@ -5,6 +5,23 @@ FactoryGirl.define do
     current_step_progress 1.5
     association :workbench
     association :referential
+    file {File.open(File.join(Rails.root, 'spec', 'fixtures', 'OFFRE_TRANSDEV_2017030112251.zip'))}
+    status :new
+    started_at nil
+    ended_at nil
+    creator 'rspec'
+
+    after(:build) do |import|
+      import.class.skip_callback(:create, :before, :initialize_fields)
+    end
+  end
+
+  factory :bad_import do
+    sequence(:name) { |n| "Import #{n}" }
+    current_step_id "MyString"
+    current_step_progress 1.5
+    association :workbench
+    association :referential
     file {File.open(File.join(Rails.root, 'spec', 'fixtures', 'terminated_job.json'))}
     status :new
     started_at nil
diff --git a/spec/factories/netex_imports.rb b/spec/factories/netex_imports.rb
index 057e47730..9e9d836e4 100644
--- a/spec/factories/netex_imports.rb
+++ b/spec/factories/netex_imports.rb
@@ -1,5 +1,5 @@
 FactoryGirl.define do
   factory :netex_import, class: NetexImport, parent: :import do
-    file { File.open(Rails.root.join('spec', 'fixtures', 'terminated_job.json')) }
+    file { File.open(Rails.root.join('spec', 'fixtures', 'OFFRE_TRANSDEV_2017030112251.zip')) }
   end
 end
diff --git a/spec/factories/workbench_imports.rb b/spec/factories/workbench_imports.rb
index 5cdcfd15f..466bfe688 100644
--- a/spec/factories/workbench_imports.rb
+++ b/spec/factories/workbench_imports.rb
@@ -1,5 +1,5 @@
 FactoryGirl.define do
   factory :workbench_import, class: WorkbenchImport, parent: :import do
-    file { File.open(Rails.root.join('spec', 'fixtures', 'terminated_job.json')) }
+    file { File.open(Rails.root.join('spec', 'fixtures', 'OFFRE_TRANSDEV_2017030112251.zip')) }
   end
 end
diff --git a/spec/fixtures/OFFRE_WITH_EXTRA.zip b/spec/fixtures/OFFRE_WITH_EXTRA.zip
new file mode 100644
index 000000000..97ea3f513
--- /dev/null
+++ b/spec/fixtures/OFFRE_WITH_EXTRA.zip
diff --git a/spec/lib/compliance_control_set_cloner_spec.rb b/spec/lib/compliance_control_set_cloner_spec.rb
new file mode 100644
index 000000000..4305ec70b
--- /dev/null
+++ b/spec/lib/compliance_control_set_cloner_spec.rb
@@ -0,0 +1,144 @@
+RSpec.describe ComplianceControlSetCloner do
+
+  subject{ described_class.new }
+
+  let( :new_organisation ){ create :organisation }
+
+  let( :source_set ){ create :compliance_control_set }
+  let( :set_prefix ){ I18n.t('compliance_control_sets.clone.prefix') }
+  let( :block_prefix ){ I18n.t('compliance_control_blocks.clone.prefix') }
+  let( :control_prefix ){ I18n.t('compliance_controls.clone.prefix') }
+
+
+  context 'Copying empty set' do
+
+    context 'correct organisation' do
+
+      #
+      #
+      #
+      #
+      #
+      #
+      #
+      #                                                 +-------------------+
+      #          +---------------------+----------------| Control (direct0) |
+      #          |                     |                +-------------------+
+      #          |                     |
+      #          |                     |                +-------------------+
+      #          +---------------------)------------+---| Control (direct1) |
+      #          |                     |            |   +-------------------+
+      #          |                     |            |
+      #          |                     |            |   +-------------------+
+      #          +---------------------)------------)---| Control (direct2) |
+      #          |                     |            |   +-------------------+
+      #          |                     |            |
+      #          |                     |            |
+      #          |                     |            |
+      #          v                     v            |
+      #   +------------+          +--------------+  |  +---------------------+
+      #   | ControlSet |<----+----| ControlBlock |<-)--| Control (indirect0) |
+      #   +------------+     |    +--------------+  |  +---------------------+
+      #                      |                      |
+      #                      |    +--------------+<-+  +---------------------+
+      #                      |<---| ControlBlock |<----| Control (indirect1) |
+      #                      |    +--------------+     +---------------------+
+      #                      |
+      #                      |    +--------------+     +---------------------+
+      #                      +----| ControlBlock |<----| Control (indirect2) |
+      #                           +--------------+     +---------------------+
+
+      context 'Directed Acyclic Graph is copied correctly' do
+        let(:source_blox){
+          3.times.map{ |_| create :compliance_control_block, compliance_control_set: source_set }
+        }
+        let(:direct_ccs){
+          3.times.map{ |n| create :generic_attribute_control_min_max, compliance_control_set: source_set, name: "direct #{n.succ}", code: "direct-#{n.succ}" }
+        }
+        # Needed to check we do not dulicate a node (compliance_control) twice
+        let(:indirect_ccs){
+          # Create 1 child for each block and also associate first of the direct ccs to the first block
+          #                                                  seconf of the direct css to the second block
+          source_blox.take(2).zip(direct_ccs.take(2)).each do | source_block, cc |
+            cc.update compliance_control_block_id: source_block.id
+          end
+          source_blox.each_with_index.map{ | source_block, n |
+            create(:generic_attribute_control_min_max, compliance_control_set: source_set, compliance_control_block: source_block, name: "indirect #{n.succ}", code: "indirect-#{n.succ}")
+          }
+        }
+        let( :sources ){ source_set.compliance_controls.order(:id) }
+
+        let( :target_set ){ ComplianceControlSet.last }
+        let( :target_blox ){ ComplianceControlBlock.last 3 }
+        let( :targets ){ target_set.compliance_controls.order(:id) }
+
+        before do
+          direct_ccs
+          indirect_ccs
+        end
+        it 'correctly creates a set for a complete DAG' do
+          # Slowness of tests constrains us to create a minimum of objects in the DB,
+          # hence only one example :(
+          #
+          #  Execute copy and keep count
+          counts = object_counts
+          subject.copy(source_set.id, new_organisation.id)
+          delta  = count_diff counts, object_counts
+
+          # Check correctly copied set
+          expect(target_set.organisation).to eq(new_organisation)
+          expect(target_set.name).to eq( [set_prefix, source_set.name].join(' ') )
+
+          # Check correctly copied controls
+          targets.zip(sources).each do | target, source |
+            expect( target.code ).to eq(source.code )
+            expect( target.comment ).to eq(source.comment )
+            expect( target.compliance_control_set ).to eq( target_set )
+            expect( target.control_attributes ).to eq(source.control_attributes)
+            expect( target.criticity ).to eq(source.criticity )
+            expect( target.name ).to eq([control_prefix, source.name].join(' '))
+            expect( target.origin_code ).to eq(source.origin_code )
+            expect( target.type ).to eq(source.type)
+          end
+          # Check correctly copied blocks
+          target_blox.zip(source_blox).each do | target_block, source_block |
+            expect( target_block.compliance_control_set ).to eq(target_set)
+            expect( target_block.name ).to eq( [block_prefix, source_block.name].join(' ') )
+            expect( target_block.condition_attributes ).to eq( source_block.condition_attributes )
+          end
+
+          # Check correct block associations
+          # See diagram above to understand the meaning of this:
+          #   - The first two controls have been assigned to the first two blocks accordingly
+          #   - The third has no block
+          #   - The last three controls have been created from the three blocks in order
+          expected_block_ids = target_blox.take(2).map(&:id) + [ nil ] + target_blox.map(&:id)
+          expect( targets.pluck(:compliance_control_block_id) ).to eq( expected_block_ids )
+
+          # Check overall counts (no additional creations)
+          expect( delta ).to eq(counts)
+        end
+      end
+
+    end
+
+    def object_counts
+      {
+        source_set_count: ComplianceControlSet.count,
+        cc_block_count: ComplianceControlBlock.count,
+        cc_count: ComplianceControl.count,
+        cck_set_count: ComplianceCheckSet.count,
+        cck_block_count: ComplianceCheckBlock.count,
+        cck_count: ComplianceCheck.count
+      }
+    end
+
+    def count_diff count1, count2
+      count1.inject({}){ |h, (k,v)|
+        h.merge( k => count2[k] - v )
+      }
+    end
+
+  end
+
+end
diff --git a/spec/models/compliance_control_validations/genric_attribute_validation/min_max_validation_spec.rb b/spec/models/compliance_control_validations/genric_attribute_validation/min_max_validation_spec.rb
index 276d09a92..4d30d61e3 100644
--- a/spec/models/compliance_control_validations/genric_attribute_validation/min_max_validation_spec.rb
+++ b/spec/models/compliance_control_validations/genric_attribute_validation/min_max_validation_spec.rb
@@ -1,8 +1,8 @@
-# RSpec.describe GenericAttributeControl::MinMax do
+RSpec.describe GenericAttributeControl::MinMax do
 
-#   let( :factory ){ :generic_attribute_control_min_max }
-#   subject{ build factory }
+  let( :factory ){ :generic_attribute_control_min_max }
+  subject{ build factory }
 
-#   it_behaves_like 'has min_max_values'
+  it_behaves_like 'has min_max_values'
 
-# end
+end
diff --git a/spec/models/import_spec.rb b/spec/models/import_spec.rb
index cd5a30982..c06d05dab 100644
--- a/spec/models/import_spec.rb
+++ b/spec/models/import_spec.rb
@@ -10,6 +10,9 @@ RSpec.describe Import, type: :model do
   it { should validate_presence_of(:workbench) }
   it { should validate_presence_of(:creator) }
 
+  it { should allow_value('file.zip').for(:file).with_message(I18n.t('activerecord.errors.models.imports.wrong_file_extension')) }
+  it { should_not allow_values('file.json', 'file.png', 'file.pdf').for(:file) }
+
   let(:workbench_import) { build_stubbed(:workbench_import) }
   let(:workbench_import_with_completed_steps) do
     workbench_import = build_stubbed(
diff --git a/spec/services/zip_service/regression_4273_spec.rb b/spec/services/zip_service/regression_4273_spec.rb
deleted file mode 100644
index 4fe0f6539..000000000
--- a/spec/services/zip_service/regression_4273_spec.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-RSpec.describe ZipService do
-  describe 'Regression Issue # 4273 https://projects.af83.io/issues/4273' do
-    let( :zip_service ){ described_class }
-    let( :unzipper ){ zip_service.new(zip_data) }
-    let( :zip_data ){ File.read zip_file }
-
-    context 'real test data' do
-      let( :subdir_names ){ %w<OFFRE_TRANSDEV_20170301122517 OFFRE_TRANSDEV_20170301122519>  }
-      let( :expected_chksums ){
-        checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'source_') } )
-      }
-
-      let( :zip_file ){ fixtures_path 'OFFRE_TRANSDEV_2017030112251.zip' }
-      #
-      # Remove potential test artefacts
-      before do
-        subdir_names.each do | subdir_name |
-          File.unlink( subdir_file subdir_name, suffix: '.zip' ) rescue nil
-          Dir.unlink( subdir_file subdir_name ) rescue nil
-        end
-      end
-
-      it "yields the correct content" do
-        subdir_contents = {}
-        # Write ZipService Streams to files and inflate them to file system
-        unzipper.subdirs.each do | subdir |
-          File.open(subdir_file( subdir.name, suffix: '.zip' ), 'wb'){ |f| f.write subdir.stream.string }
-          unzip_subdir subdir
-        end
-        # Represent the inflated file_system as a checksum tree
-        actual_checksums = 
-          checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'target/') } )
-        expect( actual_checksums ).to eq( expected_chksums )
-      end
-
-    end
-
-  end
-
-  def checksum_trees *dirs
-    dirs.flatten.inject({},&method(:checksum_tree))
-  end
-  def checksum_tree repr, dir
-    Dir.glob("#{dir}/**/*").each do |file|
-      if !File.directory?(file)
-        repr.merge!( File.basename(file) => %x{cksum #{file}}.split.first ){ |_, ov, nv| Array(ov) << nv }
-      end
-    end
-    repr
-  end
-
-  def subdir_file( subdir, prefix: 'target_', suffix: '' )
-    fixtures_path("#{prefix}#{subdir}#{suffix}")
-  end
-
-  def unzip_subdir subdir
-    %x{unzip -oqq #{subdir_file subdir.name, suffix: '.zip'} -d #{fixture_path}/target}
-  end
-end
diff --git a/spec/services/zip_service_spec.rb b/spec/services/zip_service_spec.rb
new file mode 100644
index 000000000..98cb9026d
--- /dev/null
+++ b/spec/services/zip_service_spec.rb
@@ -0,0 +1,68 @@
+RSpec.describe ZipService do
+
+  let( :zip_service ){ described_class }
+  let( :unzipper ){ zip_service.new(zip_data) }
+  let( :zip_data ){ File.read zip_file }
+
+
+  context 'correct test data' do
+    before do
+      subdir_names.each do | subdir_name |
+        File.unlink( subdir_file subdir_name, suffix: '.zip' ) rescue nil
+        Dir.unlink( subdir_file subdir_name ) rescue nil
+      end
+    end
+    let( :subdir_names ){ %w<OFFRE_TRANSDEV_20170301122517 OFFRE_TRANSDEV_20170301122519>  }
+    let( :expected_chksums ){
+      checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'source_') } )
+    }
+
+    let( :zip_file ){ fixtures_path 'OFFRE_TRANSDEV_2017030112251.zip' }
+    #
+    # Remove potential test artefacts
+
+    it 'yields the correct content' do
+      # Write ZipService Streams to files and inflate them to file system
+      unzipper.subdirs.each do | subdir |
+        expect( subdir.spurious ).to be_empty
+        File.open(subdir_file( subdir.name, suffix: '.zip' ), 'wb'){ |f| f.write subdir.stream.string }
+        unzip_subdir subdir
+      end
+      # Represent the inflated file_system as a checksum tree
+      actual_checksums = 
+        checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'target/') } )
+      expect( actual_checksums ).to eq( expected_chksums )
+    end
+
+  end
+
+  context 'test data with spurious directories' do 
+    let( :zip_file ){ fixtures_path 'OFFRE_WITH_EXTRA.zip' }
+
+    it 'returns the extra dir in the spurious field of the entry' do
+      expect( unzipper.subdirs.first.spurious ).to eq(%w{EXTRA})
+    end
+  end
+
+
+  def checksum_trees *dirs
+    dirs.flatten.inject({},&method(:checksum_tree))
+  end
+  def checksum_tree repr, dir
+    Dir.glob("#{dir}/**/*").each do |file|
+      if !File.directory?(file)
+        repr.merge!( File.basename(file) => %x{cksum #{file}}.split.first ){ |_, ov, nv| Array(ov) << nv }
+      end
+    end
+    repr
+  end
+
+  def subdir_file( subdir, prefix: 'target_', suffix: '' )
+    fixtures_path("#{prefix}#{subdir}#{suffix}")
+  end
+
+  def unzip_subdir subdir
+    %x{unzip -oqq #{subdir_file subdir.name, suffix: '.zip'} -d #{fixture_path}/target}
+  end
+end
+
diff --git a/spec/support/random.rb b/spec/support/random.rb
index 59e1a1475..0ebc2ee5e 100644
--- a/spec/support/random.rb
+++ b/spec/support/random.rb
@@ -22,6 +22,12 @@ module Support
     def random_string
       SecureRandom.urlsafe_base64
     end
+
+    def very_random(veryness=3, joiner: '-')
+      raise ArgumentError, 'not very random' unless veryness > 1
+      veryness.times.map{ SecureRandom.uuid }.join(joiner)
+    end
+
   end
 end
 
diff --git a/spec/workers/clean_up_worker_spec.rb b/spec/workers/clean_up_worker_spec.rb
index e85768fa3..fd767db00 100644
--- a/spec/workers/clean_up_worker_spec.rb
+++ b/spec/workers/clean_up_worker_spec.rb
@@ -1,4 +1,3 @@
-require 'rails_helper'
 RSpec.describe CleanUpWorker, type: :worker do
     pending "add some examples to (or delete) #{__FILE__}"
 end
diff --git a/spec/workers/compliance_control_set_cloning_worker_spec.rb b/spec/workers/compliance_control_set_cloning_worker_spec.rb
new file mode 100644
index 000000000..3a2332f62
--- /dev/null
+++ b/spec/workers/compliance_control_set_cloning_worker_spec.rb
@@ -0,0 +1,15 @@
+RSpec.describe ComplianceControlSetCloningWorker do
+
+
+  it 'is a worker' do
+    expect( described_class.new ).to be_a(Sidekiq::Worker)
+  end
+
+  it 'delegates perform to the correct lib call' do
+    id = double('id')
+    organisation_id = double('organisation_id')
+    expect_any_instance_of(ComplianceControlSetCloner).to receive(:copy).with(id, organisation_id) 
+    described_class.new.perform(id, organisation_id)
+  end
+  
+end
diff --git a/spec/workers/workbench_import/workbench_import_with_corrupt_zip_spec.rb b/spec/workers/workbench_import/workbench_import_with_corrupt_zip_spec.rb
new file mode 100644
index 000000000..5e34b208a
--- /dev/null
+++ b/spec/workers/workbench_import/workbench_import_with_corrupt_zip_spec.rb
@@ -0,0 +1,48 @@
+RSpec.describe WorkbenchImportWorker do
+
+
+  shared_examples_for 'corrupt zipfile data' do
+    subject { described_class.new }
+    let( :workbench_import ){ create :workbench_import, status: :pending }
+
+    before do
+      # Let us make sure that the name Enterprise will never be forgotten by history,
+      # ahem, I meant, that nothing is uploaded, by forbidding any message to be sent
+      # to HTTPService
+      expect_it.to receive(:download).and_return(downloaded)
+    end
+
+    it 'does not upload' do
+      stub_const 'HTTPService', double('HTTPService')
+      subject.perform(workbench_import.id)
+    end
+
+    it 'does create a message' do
+      expect{ subject.perform(workbench_import.id) }.to change{ workbench_import.messages.count }.by(1)
+
+      message = workbench_import.messages.last
+      expect( message.criticity ).to eq('error')
+      expect( message.message_key ).to eq('corrupt_zip_file')
+      expect( message.message_attributes ).to eq( 'import_name' => workbench_import.name )
+    end
+
+    it 'does not change current step' do
+      expect{ subject.perform(workbench_import.id) }.not_to change{ workbench_import.current_step }
+    end
+
+    it "sets the workbench_import.status to failed" do
+      subject.perform(workbench_import.id)
+      expect( workbench_import.reload.status ).to eq('failed')
+    end
+  end
+
+  context 'empty zip file' do 
+    let( :downloaded ){ '' }
+    it_should_behave_like 'corrupt zipfile data'
+  end
+
+  context 'corrupt data' do 
+    let( :downloaded ){ very_random }
+    it_should_behave_like 'corrupt zipfile data'
+  end
+end
diff --git a/spec/workers/workbench_import_worker_spec.rb b/spec/workers/workbench_import/workbench_import_worker_spec.rb
index a349b3433..9f860a6b3 100644
--- a/spec/workers/workbench_import_worker_spec.rb
+++ b/spec/workers/workbench_import/workbench_import_worker_spec.rb
@@ -5,7 +5,7 @@ RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do
 
   let( :workbench ){ import.workbench }
   let( :referential ){ import.referential }
-  let( :api_key ){ build_stubbed :api_key, referential: referential, token: "#{referential.id}-#{SecureRandom.hex}" }
+  let( :api_key ){ build_stubbed :api_key, referential: referential, token: "#{referential.id}-#{random_hex}" }
 
   # http://www.example.com/workbenches/:workbench_id/imports/:id/download
   let( :host ){ Rails.configuration.rails_host }
@@ -13,16 +13,17 @@ RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do
 
   let( :downloaded_zip ){ double("downloaded zip") }
   let( :download_zip_response ){ OpenStruct.new( body: downloaded_zip ) }
-  let( :download_token ){ SecureRandom.urlsafe_base64 }
-
+  let( :download_token ){ random_string }
 
   let( :upload_path ) { api_v1_netex_imports_path(format: :json) }
 
+  let( :spurious ){ [[], [], []] }
   let( :subdirs ) do
     entry_count.times.map do |i|
       ZipService::Subdir.new(
         "subdir #{i}",
-        double("subdir #{i}", rewind: 0, read: '')
+        double("subdir #{i}", rewind: 0, read: ''),
+        spurious[i]
       )
     end
   end
@@ -104,8 +105,44 @@ RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do
       expect( import ).to receive(:update).with(current_step: 3, status: 'failed')
 
       expect { worker.perform import.id }.to raise_error(StopIteration)
+    end
+  end
+
+  context 'multireferential zipfile with spurious directories' do 
+    let( :entry_count ){ 2 }
+    let( :spurious1 ){ [random_string] }
+    let( :spurious2 ){ [random_string, random_string] }
+    let( :spurious ){ [spurious1, spurious2] }
+    let( :messages ){ double('messages') }
+    let( :message_attributes ){{criticity: :warning, message_key: 'inconsistent_zip_file'}}
+    let( :message1_attributes ){ message_attributes.merge(message_attributes: {'import_name' => import.name, 'spurious_dirs' => spurious1.inspect}) }
+    let( :message2_attributes ){ message_attributes.merge(message_attributes: {'import_name' => import.name, 'spurious_dirs' => spurious2.inspect}) }
+
+    before do
+      allow(import).to receive(:messages).and_return(messages)
+    end
+
+    it 'downloads a zip file, cuts it, and uploads all pieces and adds messages' do
+
+      expect(HTTPService).to receive(:get_resource)
+        .with(host: host, path: path, params: {token: download_token})
+        .and_return( download_zip_response )
+
+      subdirs.each do |subdir|
+        mock_post subdir, post_response_ok
+      end
+
+      expect( import ).to receive(:update).with(total_steps: 2)
+      expect( import ).to receive(:update).with(current_step: 1)
+      expect( messages ).to receive(:create).with(message1_attributes)
+      expect( import ).to receive(:update).with(current_step: 2)
+      expect( messages ).to receive(:create).with(message2_attributes)
+      expect( import ).to receive(:update).with(ended_at: Time.now)
+
+      worker.perform import.id
 
     end
+    
   end
 
   def mock_post subdir, response