Merge branch 'master' into 4755-page_for_validate_referential

11 files changed, 186 insertions, 66 deletions

diff --git a/spec/controllers/compliance_controls_controller_spec.rb b/spec/controllers/compliance_controls_controller_spec.rb
index 34b27530d..61e94025d 100644
--- a/spec/controllers/compliance_controls_controller_spec.rb
+++ b/spec/controllers/compliance_controls_controller_spec.rb
@@ -44,7 +44,7 @@ RSpec.describe ComplianceControlsController, type: :controller do
   describe 'POST #update' do
     it 'should be successful' do
       post :update, compliance_control_set_id: compliance_control_set.id, id: compliance_control.id, compliance_control: compliance_control.as_json.merge(type: 'GenericAttributeControl::MinMax')
-      expect(response).to redirect_to compliance_control_set_compliance_control_path(compliance_control_set, compliance_control)
+      expect(response).to redirect_to compliance_control_set_path(compliance_control_set)
     end
   end
 
diff --git a/spec/factories/imports.rb b/spec/factories/imports.rb
index 2c53106c3..e07447b60 100644
--- a/spec/factories/imports.rb
+++ b/spec/factories/imports.rb
@@ -5,6 +5,23 @@ FactoryGirl.define do
     current_step_progress 1.5
     association :workbench
     association :referential
+    file {File.open(File.join(Rails.root, 'spec', 'fixtures', 'OFFRE_TRANSDEV_2017030112251.zip'))}
+    status :new
+    started_at nil
+    ended_at nil
+    creator 'rspec'
+
+    after(:build) do |import|
+      import.class.skip_callback(:create, :before, :initialize_fields)
+    end
+  end
+
+  factory :bad_import do
+    sequence(:name) { |n| "Import #{n}" }
+    current_step_id "MyString"
+    current_step_progress 1.5
+    association :workbench
+    association :referential
     file {File.open(File.join(Rails.root, 'spec', 'fixtures', 'terminated_job.json'))}
     status :new
     started_at nil
diff --git a/spec/factories/netex_imports.rb b/spec/factories/netex_imports.rb
index 057e47730..9e9d836e4 100644
--- a/spec/factories/netex_imports.rb
+++ b/spec/factories/netex_imports.rb
@@ -1,5 +1,5 @@
 FactoryGirl.define do
   factory :netex_import, class: NetexImport, parent: :import do
-    file { File.open(Rails.root.join('spec', 'fixtures', 'terminated_job.json')) }
+    file { File.open(Rails.root.join('spec', 'fixtures', 'OFFRE_TRANSDEV_2017030112251.zip')) }
   end
 end
diff --git a/spec/factories/workbench_imports.rb b/spec/factories/workbench_imports.rb
index 5cdcfd15f..466bfe688 100644
--- a/spec/factories/workbench_imports.rb
+++ b/spec/factories/workbench_imports.rb
@@ -1,5 +1,5 @@
 FactoryGirl.define do
   factory :workbench_import, class: WorkbenchImport, parent: :import do
-    file { File.open(Rails.root.join('spec', 'fixtures', 'terminated_job.json')) }
+    file { File.open(Rails.root.join('spec', 'fixtures', 'OFFRE_TRANSDEV_2017030112251.zip')) }
   end
 end
diff --git a/spec/fixtures/OFFRE_WITH_EXTRA.zip b/spec/fixtures/OFFRE_WITH_EXTRA.zip
new file mode 100644
index 000000000..97ea3f513
--- /dev/null
+++ b/spec/fixtures/OFFRE_WITH_EXTRA.zip
diff --git a/spec/models/import_spec.rb b/spec/models/import_spec.rb
index cd5a30982..c06d05dab 100644
--- a/spec/models/import_spec.rb
+++ b/spec/models/import_spec.rb
@@ -10,6 +10,9 @@ RSpec.describe Import, type: :model do
   it { should validate_presence_of(:workbench) }
   it { should validate_presence_of(:creator) }
 
+  it { should allow_value('file.zip').for(:file).with_message(I18n.t('activerecord.errors.models.imports.wrong_file_extension')) }
+  it { should_not allow_values('file.json', 'file.png', 'file.pdf').for(:file) }
+
   let(:workbench_import) { build_stubbed(:workbench_import) }
   let(:workbench_import_with_completed_steps) do
     workbench_import = build_stubbed(
diff --git a/spec/services/zip_service/regression_4273_spec.rb b/spec/services/zip_service/regression_4273_spec.rb
deleted file mode 100644
index 4fe0f6539..000000000
--- a/spec/services/zip_service/regression_4273_spec.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-RSpec.describe ZipService do
-  describe 'Regression Issue # 4273 https://projects.af83.io/issues/4273' do
-    let( :zip_service ){ described_class }
-    let( :unzipper ){ zip_service.new(zip_data) }
-    let( :zip_data ){ File.read zip_file }
-
-    context 'real test data' do
-      let( :subdir_names ){ %w<OFFRE_TRANSDEV_20170301122517 OFFRE_TRANSDEV_20170301122519>  }
-      let( :expected_chksums ){
-        checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'source_') } )
-      }
-
-      let( :zip_file ){ fixtures_path 'OFFRE_TRANSDEV_2017030112251.zip' }
-      #
-      # Remove potential test artefacts
-      before do
-        subdir_names.each do | subdir_name |
-          File.unlink( subdir_file subdir_name, suffix: '.zip' ) rescue nil
-          Dir.unlink( subdir_file subdir_name ) rescue nil
-        end
-      end
-
-      it "yields the correct content" do
-        subdir_contents = {}
-        # Write ZipService Streams to files and inflate them to file system
-        unzipper.subdirs.each do | subdir |
-          File.open(subdir_file( subdir.name, suffix: '.zip' ), 'wb'){ |f| f.write subdir.stream.string }
-          unzip_subdir subdir
-        end
-        # Represent the inflated file_system as a checksum tree
-        actual_checksums = 
-          checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'target/') } )
-        expect( actual_checksums ).to eq( expected_chksums )
-      end
-
-    end
-
-  end
-
-  def checksum_trees *dirs
-    dirs.flatten.inject({},&method(:checksum_tree))
-  end
-  def checksum_tree repr, dir
-    Dir.glob("#{dir}/**/*").each do |file|
-      if !File.directory?(file)
-        repr.merge!( File.basename(file) => %x{cksum #{file}}.split.first ){ |_, ov, nv| Array(ov) << nv }
-      end
-    end
-    repr
-  end
-
-  def subdir_file( subdir, prefix: 'target_', suffix: '' )
-    fixtures_path("#{prefix}#{subdir}#{suffix}")
-  end
-
-  def unzip_subdir subdir
-    %x{unzip -oqq #{subdir_file subdir.name, suffix: '.zip'} -d #{fixture_path}/target}
-  end
-end
diff --git a/spec/services/zip_service_spec.rb b/spec/services/zip_service_spec.rb
new file mode 100644
index 000000000..98cb9026d
--- /dev/null
+++ b/spec/services/zip_service_spec.rb
@@ -0,0 +1,68 @@
+RSpec.describe ZipService do
+
+  let( :zip_service ){ described_class }
+  let( :unzipper ){ zip_service.new(zip_data) }
+  let( :zip_data ){ File.read zip_file }
+
+
+  context 'correct test data' do
+    before do
+      subdir_names.each do | subdir_name |
+        File.unlink( subdir_file subdir_name, suffix: '.zip' ) rescue nil
+        Dir.unlink( subdir_file subdir_name ) rescue nil
+      end
+    end
+    let( :subdir_names ){ %w<OFFRE_TRANSDEV_20170301122517 OFFRE_TRANSDEV_20170301122519>  }
+    let( :expected_chksums ){
+      checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'source_') } )
+    }
+
+    let( :zip_file ){ fixtures_path 'OFFRE_TRANSDEV_2017030112251.zip' }
+    #
+    # Remove potential test artefacts
+
+    it 'yields the correct content' do
+      # Write ZipService Streams to files and inflate them to file system
+      unzipper.subdirs.each do | subdir |
+        expect( subdir.spurious ).to be_empty
+        File.open(subdir_file( subdir.name, suffix: '.zip' ), 'wb'){ |f| f.write subdir.stream.string }
+        unzip_subdir subdir
+      end
+      # Represent the inflated file_system as a checksum tree
+      actual_checksums = 
+        checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'target/') } )
+      expect( actual_checksums ).to eq( expected_chksums )
+    end
+
+  end
+
+  context 'test data with spurious directories' do 
+    let( :zip_file ){ fixtures_path 'OFFRE_WITH_EXTRA.zip' }
+
+    it 'returns the extra dir in the spurious field of the entry' do
+      expect( unzipper.subdirs.first.spurious ).to eq(%w{EXTRA})
+    end
+  end
+
+
+  def checksum_trees *dirs
+    dirs.flatten.inject({},&method(:checksum_tree))
+  end
+  def checksum_tree repr, dir
+    Dir.glob("#{dir}/**/*").each do |file|
+      if !File.directory?(file)
+        repr.merge!( File.basename(file) => %x{cksum #{file}}.split.first ){ |_, ov, nv| Array(ov) << nv }
+      end
+    end
+    repr
+  end
+
+  def subdir_file( subdir, prefix: 'target_', suffix: '' )
+    fixtures_path("#{prefix}#{subdir}#{suffix}")
+  end
+
+  def unzip_subdir subdir
+    %x{unzip -oqq #{subdir_file subdir.name, suffix: '.zip'} -d #{fixture_path}/target}
+  end
+end
+
diff --git a/spec/support/random.rb b/spec/support/random.rb
index 59e1a1475..0ebc2ee5e 100644
--- a/spec/support/random.rb
+++ b/spec/support/random.rb
@@ -22,6 +22,12 @@ module Support
     def random_string
       SecureRandom.urlsafe_base64
     end
+
+    def very_random(veryness=3, joiner: '-')
+      raise ArgumentError, 'not very random' unless veryness > 1
+      veryness.times.map{ SecureRandom.uuid }.join(joiner)
+    end
+
   end
 end
 
diff --git a/spec/workers/workbench_import/workbench_import_with_corrupt_zip_spec.rb b/spec/workers/workbench_import/workbench_import_with_corrupt_zip_spec.rb
new file mode 100644
index 000000000..5e34b208a
--- /dev/null
+++ b/spec/workers/workbench_import/workbench_import_with_corrupt_zip_spec.rb
@@ -0,0 +1,48 @@
+RSpec.describe WorkbenchImportWorker do
+
+
+  shared_examples_for 'corrupt zipfile data' do
+    subject { described_class.new }
+    let( :workbench_import ){ create :workbench_import, status: :pending }
+
+    before do
+      # Let us make sure that the name Enterprise will never be forgotten by history,
+      # ahem, I meant, that nothing is uploaded, by forbidding any message to be sent
+      # to HTTPService
+      expect_it.to receive(:download).and_return(downloaded)
+    end
+
+    it 'does not upload' do
+      stub_const 'HTTPService', double('HTTPService')
+      subject.perform(workbench_import.id)
+    end
+
+    it 'does create a message' do
+      expect{ subject.perform(workbench_import.id) }.to change{ workbench_import.messages.count }.by(1)
+
+      message = workbench_import.messages.last
+      expect( message.criticity ).to eq('error')
+      expect( message.message_key ).to eq('corrupt_zip_file')
+      expect( message.message_attributes ).to eq( 'import_name' => workbench_import.name )
+    end
+
+    it 'does not change current step' do
+      expect{ subject.perform(workbench_import.id) }.not_to change{ workbench_import.current_step }
+    end
+
+    it "sets the workbench_import.status to failed" do
+      subject.perform(workbench_import.id)
+      expect( workbench_import.reload.status ).to eq('failed')
+    end
+  end
+
+  context 'empty zip file' do 
+    let( :downloaded ){ '' }
+    it_should_behave_like 'corrupt zipfile data'
+  end
+
+  context 'corrupt data' do 
+    let( :downloaded ){ very_random }
+    it_should_behave_like 'corrupt zipfile data'
+  end
+end
diff --git a/spec/workers/workbench_import_worker_spec.rb b/spec/workers/workbench_import/workbench_import_worker_spec.rb
index a349b3433..deaa1e3a5 100644
--- a/spec/workers/workbench_import_worker_spec.rb
+++ b/spec/workers/workbench_import/workbench_import_worker_spec.rb
@@ -5,7 +5,7 @@ RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do
 
   let( :workbench ){ import.workbench }
   let( :referential ){ import.referential }
-  let( :api_key ){ build_stubbed :api_key, referential: referential, token: "#{referential.id}-#{SecureRandom.hex}" }
+  let( :api_key ){ build_stubbed :api_key, referential: referential, token: "#{referential.id}-#{random_hex}" }
 
   # http://www.example.com/workbenches/:workbench_id/imports/:id/download
   let( :host ){ Rails.configuration.rails_host }
@@ -13,16 +13,17 @@ RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do
 
   let( :downloaded_zip ){ double("downloaded zip") }
   let( :download_zip_response ){ OpenStruct.new( body: downloaded_zip ) }
-  let( :download_token ){ SecureRandom.urlsafe_base64 }
-
+  let( :download_token ){ random_string }
 
   let( :upload_path ) { api_v1_netex_imports_path(format: :json) }
 
+  let( :spurious ){ [[], [], []] }
   let( :subdirs ) do
     entry_count.times.map do |i|
       ZipService::Subdir.new(
         "subdir #{i}",
-        double("subdir #{i}", rewind: 0, read: '')
+        double("subdir #{i}", rewind: 0, read: ''),
+        spurious[i]
       )
     end
   end
@@ -104,8 +105,44 @@ RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do
       expect( import ).to receive(:update).with(current_step: 3, status: 'failed')
 
       expect { worker.perform import.id }.to raise_error(StopIteration)
+    end
+  end
+
+  context 'multireferential zipfile with spurious directories' do 
+    let( :entry_count ){ 2 }
+    let( :spurious1 ){ [random_string] }
+    let( :spurious2 ){ [random_string, random_string] }
+    let( :spurious ){ [spurious1, spurious2] }
+    let( :messages ){ double('messages') }
+    let( :message_attributes ){{criticity: :warning, message_key: 'inconsistent_zip_file'}}
+    let( :message1_attributes ){ message_attributes.merge(message_attributes: {'import_name' => import.name, 'spurious_dirs' => spurious1.join(', ')}) }
+    let( :message2_attributes ){ message_attributes.merge(message_attributes: {'import_name' => import.name, 'spurious_dirs' => spurious2.join(', ')}) }
+
+    before do
+      allow(import).to receive(:messages).and_return(messages)
+    end
+
+    it 'downloads a zip file, cuts it, and uploads all pieces and adds messages' do
+
+      expect(HTTPService).to receive(:get_resource)
+        .with(host: host, path: path, params: {token: download_token})
+        .and_return( download_zip_response )
+
+      subdirs.each do |subdir|
+        mock_post subdir, post_response_ok
+      end
+
+      expect( import ).to receive(:update).with(total_steps: 2)
+      expect( import ).to receive(:update).with(current_step: 1)
+      expect( messages ).to receive(:create).with(message1_attributes)
+      expect( import ).to receive(:update).with(current_step: 2)
+      expect( messages ).to receive(:create).with(message2_attributes)
+      expect( import ).to receive(:update).with(ended_at: Time.now)
+
+      worker.perform import.id
 
     end
+    
   end
 
   def mock_post subdir, response