Fixes: #5006@10h;

   Implementation of Filtering Out Referentials with foreign lines in `app/workers/workbench_import_worker.rb`

   - Using ZipService with the set of allowed lines of the organisation
   - Fixing Integration Specs of the worker
   - Refactoring, Debugging and Rebasing

3 files changed, 102 insertions, 216 deletions

diff --git a/spec/workers/workbench_import/workbench_import_with_corrupt_zip_spec.rb b/spec/workers/workbench_import/workbench_import_with_corrupt_zip_spec.rb
deleted file mode 100644
index 47626f5a1..000000000
--- a/spec/workers/workbench_import/workbench_import_with_corrupt_zip_spec.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-RSpec.describe WorkbenchImportWorker do
-
-  shared_examples_for 'corrupt zipfile data' do
-    subject { described_class.new }
-    let( :workbench_import ){ create :workbench_import, status: :pending }
-
-    before do
-      # Let us make sure that the name Enterprise will never be forgotten by history,
-      # ahem, I meant, that nothing is uploaded, by forbidding any message to be sent
-      # to HTTPService
-      expect_it.to receive(:download).and_return(downloaded)
-    end
-
-    it 'does not upload' do
-      stub_const 'HTTPService', double('HTTPService')
-      subject.perform(workbench_import.id)
-    end
-
-    it 'does create a message' do
-      expect{ subject.perform(workbench_import.id) }.to change{ workbench_import.messages.count }.by(1)
-
-      message = workbench_import.messages.last
-      expect( message.criticity ).to eq('error')
-      expect( message.message_key ).to eq('corrupt_zip_file')
-      expect( message.message_attributes ).to eq( 'source_filename' => workbench_import.file.file.file )
-    end
-
-    it 'does not change current step' do
-      expect{ subject.perform(workbench_import.id) }.not_to change{ workbench_import.current_step }
-    end
-
-    it "sets the workbench_import.status to failed" do
-      subject.perform(workbench_import.id)
-      expect( workbench_import.reload.status ).to eq('failed')
-    end
-  end
-
-  context 'empty zip file' do 
-    let( :downloaded ){ '' }
-    it_should_behave_like 'corrupt zipfile data'
-  end
-
-  context 'corrupt data' do 
-    let( :downloaded ){ very_random }
-    it_should_behave_like 'corrupt zipfile data'
-  end
-end
diff --git a/spec/workers/workbench_import/workbench_import_worker_spec.rb b/spec/workers/workbench_import/workbench_import_worker_spec.rb
deleted file mode 100644
index 47ca2b4ff..000000000
--- a/spec/workers/workbench_import/workbench_import_worker_spec.rb
+++ /dev/null
@@ -1,169 +0,0 @@
-RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do
-
-  let( :worker ) { described_class.new }
-  let( :import ){ build_stubbed :import, token_download: download_token, file: zip_file }
-
-  let( :workbench ){ import.workbench }
-  let( :referential ){ import.referential }
-  let( :api_key ){ build_stubbed :api_key, referential: referential, token: "#{referential.id}-#{random_hex}" }
-
-  # http://www.example.com/workbenches/:workbench_id/imports/:id/download
-  let( :host ){ Rails.configuration.rails_host }
-  let( :path ){ download_workbench_import_path(workbench, import) }
-
-  let( :downloaded_zip ){ double("downloaded zip") }
-  let( :download_zip_response ){ OpenStruct.new( body: downloaded_zip ) }
-  let( :download_token ){ random_string }
-
-  let( :upload_path ) { api_v1_netex_imports_path(format: :json) }
-
-  let( :spurious ){ [[], [], []] }
-  let( :subdirs ) do
-    entry_count.times.map do |i|
-      ZipService::Subdir.new(
-        "subdir #{i}",
-        double("subdir #{i}", rewind: 0, read: ''),
-        spurious[i]
-      )
-    end
-  end
-
-  let( :zip_service ){ double("zip service") }
-  let( :zip_file ){ open_fixture('multiple_references_import.zip') }
-
-  let( :post_response_ok ){ double(status: 201, body: "{}") }
-
-  before do
-    Timecop.freeze(Time.now)
-
-    # Silence Logger
-    allow_any_instance_of(Logger).to receive(:info)
-    allow_any_instance_of(Logger).to receive(:warn)
-
-    # That should be `build_stubbed's` job, no?
-    allow(Import).to receive(:find).with(import.id).and_return(import)
-
-    allow(Api::V1::ApiKey).to receive(:from).and_return(api_key)
-    allow(ZipService).to receive(:new).with(downloaded_zip).and_return zip_service
-    expect(zip_service).to receive(:subdirs).and_return(subdirs)
-    expect( import ).to receive(:update).with(
-      status: 'running',
-      started_at: Time.now
-    )
-  end
-
-  after do
-    Timecop.return
-  end
-
-
-  context 'multireferential zipfile, no errors' do
-    let( :entry_count ){ 2 }
-
-    it 'downloads a zip file, cuts it, and uploads all pieces' do
-
-      expect(HTTPService).to receive(:get_resource)
-        .with(host: host, path: path, params: {token: download_token})
-        .and_return( download_zip_response )
-
-      subdirs.each do |subdir|
-        mock_post subdir, post_response_ok
-      end
-
-      expect( import ).to receive(:update).with(total_steps: 2)
-      expect( import ).to receive(:update).with(current_step: 1)
-      expect( import ).to receive(:update).with(current_step: 2)
-      expect( import ).to receive(:update).with(ended_at: Time.now)
-
-      worker.perform import.id
-
-    end
-  end
-
-  context 'multireferential zipfile with error' do
-    let( :entry_count ){ 3 }
-    let( :post_response_failure ){ double(status: 406, body: {error: 'What was you thinking'}) }
-
-    it 'downloads a zip file, cuts it, and uploads some pieces' do
-      expect(HTTPService).to receive(:get_resource)
-        .with(host: host, path: path, params: {token: download_token})
-        .and_return( download_zip_response )
-
-      # First subdir succeeds
-      subdirs[0..0].each do |subdir|
-        mock_post subdir, post_response_ok
-      end
-
-      # Second subdir fails (M I S E R A B L Y)
-      subdirs[1..1].each do |subdir|
-        mock_post subdir, post_response_failure
-      end
-
-      expect( import ).to receive(:update).with(total_steps: 3)
-      expect( import ).to receive(:update).with(current_step: 1)
-      expect( import ).to receive(:update).with(current_step: 2)
-      expect( import ).to receive(:update).with(current_step: 3, status: 'failed')
-
-      expect { worker.perform import.id }.to raise_error(StopIteration)
-    end
-  end
-
-  context 'multireferential zipfile with spurious directories' do 
-    let( :entry_count ){ 2 }
-    let( :spurious1 ){ [random_string] }
-    let( :spurious2 ){ [random_string, random_string] }
-    let( :spurious ){ [spurious1, spurious2] }
-    let( :messages ){ double('messages') }
-    let( :message_attributes ){{criticity: :warning, message_key: 'inconsistent_zip_file'}}
-    let( :message1_attributes ){ message_attributes.merge(message_attributes: {'source_filename' => import.file.file.file, 'spurious_dirs' => spurious1.join(', ')}) }
-    let( :message2_attributes ){ message_attributes.merge(message_attributes: {'source_filename' => import.file.file.file, 'spurious_dirs' => spurious2.join(', ')}) }
-
-    before do
-      allow(import).to receive(:messages).and_return(messages)
-    end
-
-    it 'downloads a zip file, cuts it, and uploads all pieces and adds messages' do
-
-      expect(HTTPService).to receive(:get_resource)
-        .with(host: host, path: path, params: {token: download_token})
-        .and_return( download_zip_response )
-
-      subdirs.each do |subdir|
-        mock_post subdir, post_response_ok
-      end
-
-      expect( import ).to receive(:update).with(total_steps: 2)
-      expect( import ).to receive(:update).with(current_step: 1)
-      expect( messages ).to receive(:create).with(message1_attributes)
-      expect( import ).to receive(:update).with(current_step: 2)
-      expect( messages ).to receive(:create).with(message2_attributes)
-      expect( import ).to receive(:update).with(ended_at: Time.now)
-
-      worker.perform import.id
-
-    end
-    
-  end
-
-  def mock_post subdir, response
-    allow(HTTPService).to receive(:upload)
-    expect( HTTPService ).to receive(:post_resource)
-      .with(
-        host: host,
-        path: upload_path,
-        params: {
-          netex_import: {
-            parent_id: import.id,
-            parent_type: import.class.name,
-            workbench_id: workbench.id,
-            name: subdir.name,
-            file: HTTPService.upload(
-              subdir.stream,
-              'application/zip',
-              "#{subdir.name}.zip"
-            )
-          }
-        }
-      ).and_return(response)
-  end
-end
diff --git a/spec/workers/workbench_import_worker_spec.rb b/spec/workers/workbench_import_worker_spec.rb
new file mode 100644
index 000000000..310693e1e
--- /dev/null
+++ b/spec/workers/workbench_import_worker_spec.rb
@@ -0,0 +1,102 @@
+RSpec.describe WorkbenchImportWorker, type: [:worker, :request, :zip] do
+
+  def self.expect_upload_with *entry_names, &blk
+    let(:expected_upload_names){ Set.new(entry_names.flatten) }
+
+    it "uploads the following entries: #{entry_names.flatten.inspect}" do
+      allow( HTTPService ).to receive(:post_resource)
+        .with(host: host, path: upload_path, params: anything) { |params|
+        name =  params[:params][:netex_import][:name]
+        raise RuntimeError, "unexpected upload of entry #{name}" unless expected_upload_names.delete?(name)
+        OpenStruct.new(status: 201)
+      }
+      instance_eval(&blk)
+      expect( expected_upload_names ).to be_empty, "the following expected uploads were not executed: #{expected_upload_names.to_a.inspect}"
+    end
+  end
+
+  let( :lines ){ %w{*:C00109 *:C00108}.to_json }
+  let!( :organisation ){ workbench.organisation.update sso_attributes: {'functional_scope' => lines}}
+
+  let( :worker ) { described_class.new }
+  let( :workbench_import ){ create :workbench_import, token_download: download_token }
+
+  let( :workbench ){ workbench_import.workbench }
+
+  # http://www.example.com/workbenches/:workbench_id/imports/:id/download
+  let( :host ){ Rails.configuration.rails_host }
+  let( :path ){ download_workbench_import_path(workbench, workbench_import) }
+  let( :upload_path ){ api_v1_netex_imports_path(format: :json) }
+
+  let( :downloaded_zip_archive ){ make_zip_from_tree zip_data_dir }
+  let( :downloaded_zip_data ){ downloaded_zip_archive.data }
+  let( :download_token ){ random_string }
+
+  before do
+    stub_request(:get, "#{ host }#{ path }?token=#{ workbench_import.token_download }"). 
+      to_return(body: downloaded_zip_data, status: :success)
+  end
+
+  context 'correct workbench_import' do
+    let( :zip_data_dir ){ fixtures_path 'two_referentials_ok' }
+
+    expect_upload_with %w{ OFFRE_TRANSDEV_20170301122517 OFFRE_TRANSDEV_20170301122519 } do
+      expect{ worker.perform( workbench_import.id ) }.not_to change{ workbench_import.messages.count }
+      expect( workbench_import.reload.attributes.values_at(*%w{current_step total_steps}) )
+        .to eq([2, 2])
+      expect( workbench_import.reload.status ).to eq('running')
+    end
+
+  end
+
+  context 'correct but spurious directories' do
+    let( :zip_data_dir ){ fixtures_path 'extra_file_nok' }
+
+    expect_upload_with [] do
+      expect{ worker.perform( workbench_import.id ) }.to change{ workbench_import.messages.count }.by(1)
+      expect( workbench_import.reload.attributes.values_at(*%w{current_step total_steps}) )
+        .to eq([0, 0])
+      expect( workbench_import.messages.last.message_key ).to eq('inconsistent_zip_file')
+      expect( workbench_import.reload.status ).to eq('running')
+    end
+  end
+
+  context 'foreign lines' do 
+    let( :zip_data_dir ){ fixtures_path 'some_foreign_mixed' }
+
+    expect_upload_with %w{ OFFRE_TRANSDEV_20170301122517 OFFRE_TRANSDEV_20170301122519 } do
+      expect{ worker.perform( workbench_import.id ) }.to change{ workbench_import.messages.count }.by(1)
+      expect( workbench_import.reload.attributes.values_at(*%w{current_step total_steps}) )
+        .to eq([2, 2])
+      expect( workbench_import.messages.last.message_key ).to eq('foreign_lines_in_referential')
+      expect( workbench_import.reload.status ).to eq('running')
+    end
+    
+  end
+
+  context 'foreign and spurious' do
+    let( :zip_data_dir ){ fixtures_path 'foreign_and_spurious' }
+
+    expect_upload_with %w{ OFFRE_TRANSDEV_20170301122517 OFFRE_TRANSDEV_20170301122519 } do
+      expect{ worker.perform( workbench_import.id ) }.to change{ workbench_import.messages.count }.by(2)
+      expect( workbench_import.reload.attributes.values_at(*%w{current_step total_steps}) )
+        .to eq([2, 2])
+      expect( workbench_import.messages.last(2).map(&:message_key).sort )
+        .to eq(%w{foreign_lines_in_referential inconsistent_zip_file})
+      expect( workbench_import.reload.status ).to eq('running')
+    end
+  end
+
+  context 'corrupt zip file' do 
+    let( :downloaded_zip_archive ){ OpenStruct.new(data: '') }
+
+    it 'will not upload anything' do
+      expect(HTTPService).not_to receive(:post_resource)
+      expect{ worker.perform( workbench_import.id ) }.to change{ workbench_import.messages.count }.by(1)
+      expect( workbench_import.messages.last.message_key ).to eq('corrupt_zip_file')
+      expect( workbench_import.reload.status ).to eq('failed')
+    end
+    
+  end
+
+end