Merge pull request #124 from af83/5006-wb_import_filter_refs_with_foreign_lines

5006 wb import filter refs with foreign lines

2 files changed, 179 insertions, 43 deletions

diff --git a/spec/services/meta_zip_data_spec.rb b/spec/services/meta_zip_data_spec.rb
new file mode 100644
index 000000000..1cbfee008
--- /dev/null
+++ b/spec/services/meta_zip_data_spec.rb
@@ -0,0 +1,55 @@
+# Convenience meta spec to debug potential bugs in zip support helpers
+# uncomment run and check files in `zip_fixtures_path`
+#
+# It also describes what the two helpers do and therefore facilitates the usage of 
+#
+#   * `make_zip` and
+#   * `make_zip_from_tree
+# 
+RSpec.describe 'ZipData', type: [:zip, :meta] do
+
+  # let( :zip_file ){ zip_fixtures_path('xxx.zip') }
+  # let( :tmp_output ){ zip_fixtures_path('tmp') }
+
+  # before do
+  #   clear_all_zip_fixtures!
+  #   Dir.mkdir(tmp_output)
+  # end
+
+  # context 'a simple archive' do
+  #   let( :zip_data ){ make_zip "xxx.zip", archive_content }
+  #   let( :archive_content ){ {
+  #         'hello.txt' => 'hello',
+  #         'subdir/too.txt' => 'in a subdir'
+  #   } }
+
+  #   it 'handmade: plausibility and manual check' do
+  #     zip_data.write_to(zip_file)
+  #     %x{unzip -oqq #{zip_file} -d #{tmp_output}}
+  #     archive_content.each do | rel_path, content |
+  #       expect(File.read(File.join(tmp_output, rel_path))).to eq(content)
+  #     end
+  #   end
+  # end
+
+  # context 'archive from dir tree' do
+  #   let( :dir ){ fixtures_path 'meta_zip' }
+  #   let( :zip_data ){ make_zip_from_tree dir }
+
+  #   let( :archive_content ){ {
+  #     'one/alpha'        => "alpha\n",
+  #     'two/beta'         => "beta\n",
+  #     'two/subdir/gamma' => "gamma\n"
+  #   } }
+
+  #   it 'directory: plausibility and manual check' do
+  #     zip_data.write_to(zip_file)
+  #     %x{unzip -oqq #{zip_file} -d #{tmp_output}}
+  #     archive_content.each do | rel_path, content |
+  #       expect(File.read(File.join(tmp_output, rel_path))).to eq(content)
+  #     end
+  #   end
+
+  # end
+  
+end
diff --git a/spec/services/zip_service_spec.rb b/spec/services/zip_service_spec.rb
index 98cb9026d..751e6c3f1 100644
--- a/spec/services/zip_service_spec.rb
+++ b/spec/services/zip_service_spec.rb
@@ -1,68 +1,149 @@
-RSpec.describe ZipService do
+RSpec.describe ZipService, type: :zip do
 
-  let( :zip_service ){ described_class }
-  let( :unzipper ){ zip_service.new(zip_data) }
-  let( :zip_data ){ File.read zip_file }
+  let( :unzipper ){ described_class.new( zip_data, allowed_lines ) }
+  subject { unzipper.subdirs }
 
+  let( :allowed_lines ){ Set.new(%w{C00108 C00109}) }
+  let( :zip_data ){ zip_archive.data }
+  let( :zip_archive ){ make_zip zip_name, zip_content }
 
-  context 'correct test data' do
-    before do
-      subdir_names.each do | subdir_name |
-        File.unlink( subdir_file subdir_name, suffix: '.zip' ) rescue nil
-        Dir.unlink( subdir_file subdir_name ) rescue nil
+  context 'one, legal, referential' do
+    let( :zip_name ){ 'one_referential_ok.zip' }
+    let( :zip_content ){ first_referential_ok_data }
+
+    it 'yields correct output' do
+      subject.each do | subdir |
+        expect_correct_subdir subdir, make_zip('expected.zip', first_referential_ok_data)
       end
     end
-    let( :subdir_names ){ %w<OFFRE_TRANSDEV_20170301122517 OFFRE_TRANSDEV_20170301122519>  }
-    let( :expected_chksums ){
-      checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'source_') } )
-    }
+  end
 
-    let( :zip_file ){ fixtures_path 'OFFRE_TRANSDEV_2017030112251.zip' }
-    #
-    # Remove potential test artefacts
+  context 'two, legal, referentials' do
+    let( :zip_name ){ 'two_referential_ok.zip' }
+    let( :zip_content ){ first_referential_ok_data.merge( second_referential_ok_data ) }
+    let( :expected_zips ){ [
+      make_zip('expected.zip', first_referential_ok_data),
+      make_zip('expected.zip', second_referential_ok_data)
+    ] }
 
-    it 'yields the correct content' do
-      # Write ZipService Streams to files and inflate them to file system
-      unzipper.subdirs.each do | subdir |
-        expect( subdir.spurious ).to be_empty
-        File.open(subdir_file( subdir.name, suffix: '.zip' ), 'wb'){ |f| f.write subdir.stream.string }
-        unzip_subdir subdir
+    it 'yields correct output' do
+      subject.zip(expected_zips).each do | subdir, expected_zip |
+        expect_correct_subdir subdir, expected_zip
       end
-      # Represent the inflated file_system as a checksum tree
-      actual_checksums = 
-        checksum_trees( subdir_names.map{ |sn| subdir_file(sn, prefix: 'target/') } )
-      expect( actual_checksums ).to eq( expected_chksums )
     end
+  end
 
+  context 'one referential with a spurious directory' do
+    let( :zip_name ){ 'one_referential_spurious.zip' }
+    let( :zip_content ){ first_referential_spurious_data }
+
+    it 'returns a not ok object' do
+      expect_incorrect_subdir subject.first, expected_spurious: %w{SPURIOUS}
+    end
   end
 
-  context 'test data with spurious directories' do 
-    let( :zip_file ){ fixtures_path 'OFFRE_WITH_EXTRA.zip' }
+  context 'one referential with a foreign line' do
+    let( :zip_name ){ 'one_referential_foreign.zip' }
+    let( :zip_content ){ first_referential_foreign_data }
 
-    it 'returns the extra dir in the spurious field of the entry' do
-      expect( unzipper.subdirs.first.spurious ).to eq(%w{EXTRA})
+    it 'returns a not ok object' do
+      expect_incorrect_subdir subject.first, expected_foreign_lines: %w{C00110}
     end
   end
 
+  context '1st ref ok, 2nd foreign line, 3rd spurious' do
+    let( :zip_name ){ '3-mixture.zip' }
+    let( :zip_content ){ first_referential_ok_data
+                           .merge(first_referential_foreign_data)
+                           .merge(first_referential_spurious_data) }
+
+    it 'returns 3 objects accordingly' do
+      subdirs = subject.to_a
 
-  def checksum_trees *dirs
-    dirs.flatten.inject({},&method(:checksum_tree))
+      expect_correct_subdir subdirs.first, make_zip('expected.zip', first_referential_ok_data)
+
+      expect_incorrect_subdir subdirs.second, expected_foreign_lines: %w{C00110}
+      expect_incorrect_subdir subdirs.third,  expected_spurious: %W{SPURIOUS}
+    end
   end
-  def checksum_tree repr, dir
-    Dir.glob("#{dir}/**/*").each do |file|
-      if !File.directory?(file)
-        repr.merge!( File.basename(file) => %x{cksum #{file}}.split.first ){ |_, ov, nv| Array(ov) << nv }
-      end
+
+  context 'one messed up' do 
+    let( :zip_name ){ 'one_messed_up.zip' }
+    let( :zip_content ){ messed_up_referential_data }
+
+    it 'returns a not ok object (all error information provided)' do
+      expect_incorrect_subdir subject.first,
+                                expected_foreign_lines: %w{C00110 C00111},
+                                expected_spurious: %w{SPURIOUS1 SPURIOUS2}
+    end
+    
+  end
+  # Behaviour
+  # ---------
+  def expect_correct_subdir subdir, expected_zip 
+    expect( subdir ).to be_ok
+    expect( subdir.foreign_lines ).to be_empty
+    expect( subdir.spurious ).to be_empty
+    subdir.stream.tap do | stream |
+      stream.rewind
+      expect( stream.read ).to eq(expected_zip.data)
     end
-    repr
   end
 
-  def subdir_file( subdir, prefix: 'target_', suffix: '' )
-    fixtures_path("#{prefix}#{subdir}#{suffix}")
+  def expect_incorrect_subdir subdir, expected_spurious: [], expected_foreign_lines: []
+    expect( subdir ).not_to be_ok
+    expect( subdir.foreign_lines ).to eq(expected_foreign_lines)
+    expect( subdir.spurious ).to eq(expected_spurious)
   end
 
-  def unzip_subdir subdir
-    %x{unzip -oqq #{subdir_file subdir.name, suffix: '.zip'} -d #{fixture_path}/target}
+  # Data
+  # ----
+  let :first_referential_ok_data do
+    {
+       'Referential1/calendriers.xml'     => 'calendriers',
+       'Referential1/commun.xml'          => 'common',
+       'Referential1/offre_C00108_9.xml'  => 'line 108 ref 1',
+       'Referential1/offre_C00109_10.xml' => 'line 109 ref 1'
+    }
   end
-end
+  let :first_referential_foreign_data do
+    {
+       'Referential2/calendriers.xml'     => 'calendriers',
+       'Referential2/commun.xml'          => 'common',
+       'Referential2/offre_C00110_11.xml' => 'foreign line ref 1',
+       'Referential2/offre_C00108_9.xml'  => 'line 108 ref 1',
+       'Referential2/offre_C00109_10.xml' => 'line 109 ref 1'
+    }
+  end
+  let :first_referential_spurious_data do
+    {
+       'Referential3/calendriers.xml'     => 'calendriers',
+       'Referential3/commun.xml'          => 'common',
+       'Referential3/SPURIOUS/commun.xml' => 'common',
+       'Referential3/offre_C00108_9.xml'  => 'line 108 ref 1',
+       'Referential3/offre_C00109_10.xml' => 'line 109 ref 1'
+    }
+  end
+  let :second_referential_ok_data do
+    {
+       'Referential4/calendriers.xml'     => 'calendriers',
+       'Referential4/commun.xml'          => 'common',
+       'Referential4/offre_C00108_9.xml'  => 'line 108 ref 2',
+       'Referential4/offre_C00109_10.xml' => 'line 109 ref 2'
+    }
+  end
+  let :messed_up_referential_data do
+    {
+       'Referential5/calendriers.xml'      => 'calendriers',
+       'Referential5/commun.xml'           => 'common',
+       'Referential5/SPURIOUS1/commun.xml' => 'common',
+       'Referential5/SPURIOUS2/commun.xml' => 'common',
+       'Referential5/offre_C00110_11.xml'  => 'foreign line ref 1',
+       'Referential5/offre_C00111_11.xml'  => 'foreign line ref 1',
+       'Referential5/offre_C00108_9.xml'   => 'line 108 ref 1',
+       'Referential5/offre_C00109_10.xml'  => 'line 109 ref 1'
+    }
+  end
+
 
+end