Refs: #3507;20h Specing and Implementing the ZipService

24 files changed, 317 insertions, 28 deletions

diff --git a/app/concerns/configurable.rb b/app/concerns/configurable.rb
new file mode 100644
index 000000000..c7d0f1fd9
--- /dev/null
+++ b/app/concerns/configurable.rb
@@ -0,0 +1,26 @@
+module Configurable
+  
+  module ClassMethods
+    def config &blk
+      blk ? blk.(configuration) : configuration
+    end
+
+    private
+    def configuration
+      @__configuration__ ||= Rails::Application::Configuration.new
+    end
+  end
+
+  module InstanceMethods
+    private
+
+    def config
+      self.class.config
+    end
+  end
+
+  def self.included(into)
+    into.extend ClassMethods
+    into.send :include, InstanceMethods
+  end
+end
diff --git a/app/services/file_service.rb b/app/services/file_service.rb
new file mode 100644
index 000000000..efccbe24f
--- /dev/null
+++ b/app/services/file_service.rb
@@ -0,0 +1,23 @@
+module FileService extend self
+
+  def unique_filename( path, enum_with: with_ints )
+    file_names = enum_with.map( &file_name_maker(path) )
+    file_names
+      .drop_while( &File.method(:exists?) )
+      .next
+  end
+
+  def with_ints(format='%d')
+    (0..Float::INFINITY)
+      .lazy
+      .map{ |n| format % n }
+  end
+  
+
+  private
+
+  def file_name_maker path
+    ->(n){ [path, n].join('_') }
+  end
+
+end
diff --git a/app/services/http_service.rb b/app/services/http_service.rb
new file mode 100644
index 000000000..a3c4d2569
--- /dev/null
+++ b/app/services/http_service.rb
@@ -0,0 +1,36 @@
+class HTTPService
+  def self.get_resource(*args)
+    new.get_resource(*args)
+  end
+
+  def get_resource(host:, path:, token: nil, params: {}, parse_json: false) 
+    Faraday.new(url: host) do |c|
+      c.headers['Authorization'] = "Token token=#{token.inspect}" if token
+      c.adapter Faraday.default_adapter
+
+      resp = c.get path, params
+      if resp.status == 200
+        return parse_json ? JSON.parse(resp.body) : resp.body
+      else
+        raise "Error on api request status : #{resp.status} => #{resp.body}"
+      end
+    end
+  end
+
+  def post_resource(host:, path:, resource_name:, token: nil, params: {}, upload: nil)
+    Faraday.new(url: host) do |c|
+      c.headers['Authorization'] = "Token token=#{token.inspect}" if token
+      c.request :multipart
+      c.request :url_encoded
+      c.adapter Faraday.default_adapter
+
+      if upload
+        name = upload.keys.first
+        value, mime_type, as_name = upload.values.first
+        params.update( name => Faraday::UploadIO.new(value, mime_type, as_name ) )
+      end
+
+      c.post path, resource_name => params
+    end
+  end
+end
diff --git a/app/services/zip_service.rb b/app/services/zip_service.rb
new file mode 100644
index 000000000..778bfd06d
--- /dev/null
+++ b/app/services/zip_service.rb
@@ -0,0 +1,55 @@
+class ZipService
+
+  attr_reader :current_entry, :zip_data
+
+  def initialize data
+    @zip_data = data
+    @current_entry = nil
+  end
+
+  class << self
+    def convert_entries entries
+      -> output_stream do
+        entries.each do |e|
+          output_stream.put_next_entry e.name
+          output_stream.write e.get_input_stream.read
+        end
+      end
+    end
+
+    def entries input_stream
+      Enumerator.new do |enum|
+        loop{ enum << input_stream.get_next_entry }
+      end.lazy.take_while{ |e| e }
+    end
+  end
+
+  def entry_groups
+    self.class.entries(input_stream).group_by(&method(:entry_key))
+  end
+
+  def entry_group_streams
+    entry_groups.map(&method(:make_stream)).to_h
+  end
+
+  def entry_key entry
+    entry.name.split('/', -1)[-2]
+  end
+
+  def make_stream pair
+    name, entries = pair 
+    [name,  make_stream_from( entries )]
+  end
+
+  def make_stream_from entries
+    Zip::OutputStream.write_buffer(&self.class.convert_entries(entries))
+  end
+
+  def next_entry
+    @current_entry = input_stream.get_next_entry
+  end
+
+  def input_stream
+    @__input_stream__ ||= Zip::InputStream.open(StringIO.new(zip_data))
+  end
+end
diff --git a/app/workers/workbench_import_worker.rb b/app/workers/workbench_import_worker.rb
index 18d3ae112..d1a381e0c 100644
--- a/app/workers/workbench_import_worker.rb
+++ b/app/workers/workbench_import_worker.rb
@@ -1,6 +1,7 @@
 class WorkbenchImportWorker
   include Sidekiq::Worker
   include Rails.application.routes.url_helpers
+  include Configurable
 
   attr_reader :import, :downloaded
 
@@ -17,18 +18,22 @@ class WorkbenchImportWorker
       path: import_path,
       params: {token: import.token_download})
 
-    Tempfile.open( do | tmpfile |
-      tmpfile.write zipfile_data
-      @downloaded = tmpfile.path
+    path = File.join(config.dir, import.name.gsub(%r{\s+}, '-'))
+    unique_path = FileService.unique_filename path
+    Dir.mkdir unique_path
+    @downloaded = File.join(unique_path, import.name)
+    File.open(downloaded, 'wb') do | file |
+      file.write zipfile_data
     end
 
-    if one_entry?
-      upload(@downloaded)
+    if single_entry?
+      upload(downloaded)
     else
       split_zip.each(&method(:upload))
     end
   end
 
+
   def single_entry?
     true
   end
@@ -40,6 +45,12 @@ class WorkbenchImportWorker
   def upload zip_file
   end
 
+  # Memoized Values
+
+  def dirname
+    @__dirname__ ||= make_unique_dir
+  end
+
   def import_host
     @__import_host__ ||= Rails.application.config.front_end_host
   end
diff --git a/config/application.rb b/config/application.rb
index 910ddd983..e2f6f8e7b 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -14,7 +14,8 @@ module ChouetteIhm
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.
-    config.autoload_paths << config.root.join("lib")
+    config.autoload_paths << config.root.join('lib')
+    config.autoload_paths << config.root.join('concerns')
 
     # custom exception pages
     config.exceptions_app = self.routes
diff --git a/config/environments/development.rb b/config/environments/development.rb
index 42523a761..1384d0c00 100644
--- a/config/environments/development.rb
+++ b/config/environments/development.rb
@@ -91,5 +91,7 @@ Rails.application.configure do
   # link to validation specification pages
   config.validation_spec = "http://www.chouette.mobi/neptune-validation/v21/"
 
+  # Local zip decompression dir
+  #
   config.i18n.available_locales = [:fr, :en]
 end
diff --git a/config/initializers/workbench_import.rb b/config/initializers/workbench_import.rb
new file mode 100644
index 000000000..1e405c9ca
--- /dev/null
+++ b/config/initializers/workbench_import.rb
@@ -0,0 +1,5 @@
+WorkbenchImportWorker.config do | config |
+  config.dir = ENV.fetch('WORKBENCH_IMPORT_DIR'){ Rails.root.join 'tmp/workbench_import' }
+
+  FileUtils.mkdir_p config.dir rescue nil
+end
diff --git a/hello_world b/hello_world
new file mode 100644
index 000000000..3b18e512d
--- /dev/null
+++ b/hello_world
@@ -0,0 +1 @@
+hello world
diff --git a/lib/af83/http_fetcher.rb b/lib/af83/http_fetcher.rb
deleted file mode 100644
index 514dd1dc9..000000000
--- a/lib/af83/http_fetcher.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-module AF83
-  class HTTPFetcher
-    def self.get_resource(*args)
-      new.get_resource(*args)
-    end
-
-    def get_resource(host:, path:, token: nil, params: {}, parse_json: false) 
-      Faraday.new(url: host) do |c|
-        c.headers['Authorization'] = "Token token=#{token.inspect}" if token
-        c.adapter  Faraday.default_adapter
-
-        resp = c.get path, params
-        if resp.status == 200
-          return parse_json ? JSON.parse(resp.body) : resp.body
-        else
-          raise "Error on api request status : #{resp.status} => #{resp.body}"
-        end
-      end
-    end
-  end
-end
diff --git a/spec/concerns/configurable_spec.rb b/spec/concerns/configurable_spec.rb
new file mode 100644
index 000000000..822f572c1
--- /dev/null
+++ b/spec/concerns/configurable_spec.rb
@@ -0,0 +1,31 @@
+RSpec.describe Configurable do
+
+  subject do
+    Class.new do
+      include Configurable
+    end
+  end
+
+  let( :something ){ double('something') }
+
+  it 'can be configured' do
+    expect{ subject.config.anything }.to raise_error(NoMethodError)
+
+    subject.config.something = something
+
+    expect( subject.config.something ).to eq(something)
+    expect( subject.new.send(:config).something ).to eq(something)
+    expect( subject.new.send(:config).something ).to eq(something)
+  end
+
+  it 'can be configured with a block' do
+
+    subject.config do | c |
+      c.something = something 
+    end
+
+    expect( subject.config.something ).to eq(something)
+    expect( subject.new.send(:config).something ).to eq(something)
+    expect( subject.new.send(:config).something ).to eq(something)
+  end
+end
diff --git a/spec/fixtures/multiref.zip b/spec/fixtures/multiref.zip
index 5b28f7cad..28ddff198 100644
--- a/spec/fixtures/multiref.zip
+++ b/spec/fixtures/multiref.zip
diff --git a/spec/fixtures/nozip.zip b/spec/fixtures/nozip.zip
new file mode 100644
index 000000000..505bd213a
--- /dev/null
+++ b/spec/fixtures/nozip.zip
@@ -0,0 +1 @@
+no zip file
diff --git a/spec/fixtures/ref1.zip b/spec/fixtures/ref1.zip
new file mode 100644
index 000000000..1cbd0268e
--- /dev/null
+++ b/spec/fixtures/ref1.zip
diff --git a/spec/fixtures/ref2.zip b/spec/fixtures/ref2.zip
new file mode 100644
index 000000000..342353b07
--- /dev/null
+++ b/spec/fixtures/ref2.zip
diff --git a/spec/fixtures/singleref.zip b/spec/fixtures/singleref.zip
new file mode 100644
index 000000000..4aee23614
--- /dev/null
+++ b/spec/fixtures/singleref.zip
diff --git a/spec/fixtures/uniref.zip b/spec/fixtures/uniref.zip
deleted file mode 100644
index 86b688b51..000000000
--- a/spec/fixtures/uniref.zip
+++ /dev/null
diff --git a/spec/requests/api/v1/netex_import_spec.rb b/spec/requests/api/v1/netex_import_spec.rb
index e67cef9a2..1c34d5544 100644
--- a/spec/requests/api/v1/netex_import_spec.rb
+++ b/spec/requests/api/v1/netex_import_spec.rb
@@ -4,7 +4,7 @@ RSpec.describe "NetexImport", type: :request do
 
     let( :referential ){ create :referential }
 
-    let( :file_path ){'spec/fixtures/neptune.zip'}
+    let( :file_path ){ fixtures_path 'singleref.zip' }
     let( :file ){ fixture_file_upload( file_path ) }
 
     let( :post_request ) do
diff --git a/spec/services/file_service_spec.rb b/spec/services/file_service_spec.rb
new file mode 100644
index 000000000..90e164408
--- /dev/null
+++ b/spec/services/file_service_spec.rb
@@ -0,0 +1,16 @@
+RSpec.describe FileService do
+
+  it 'computes a unique filename' do
+    expect( File ).to receive(:exists?).with('xxx/yyy_0').and_return( false )
+
+    expect(described_class.unique_filename('xxx/yyy')).to eq('xxx/yyy_0')
+  end
+
+  it 'handles duplicate names by means of a counter' do
+    expect( File ).to receive(:exists?).with('xxx/yyy_0').and_return( true )
+    expect( File ).to receive(:exists?).with('xxx/yyy_1').and_return( true )
+    expect( File ).to receive(:exists?).with('xxx/yyy_2').and_return( false )
+
+    expect(described_class.unique_filename('xxx/yyy')).to eq('xxx/yyy_2')
+  end
+end
diff --git a/spec/services/zip_service/zip_entry_data_spec.rb b/spec/services/zip_service/zip_entry_data_spec.rb
new file mode 100644
index 000000000..6bfaa8cc4
--- /dev/null
+++ b/spec/services/zip_service/zip_entry_data_spec.rb
@@ -0,0 +1,35 @@
+RSpec.describe ZipService do
+
+  subject{ described_class.new(read_fixture('multiref.zip')) }
+
+  let( :ref1_zipdata ){ read_fixture('ref1.zip') }
+  let( :ref2_zipdata ){ read_fixture('ref2.zip') }
+
+  it 'can group all entries' do
+    expect( subject.entry_groups.keys ).to eq(%w{ref1 ref2})
+  end
+
+  context 'creates correct zip data for each subdir' do
+    it 'e.g. ref1' do
+      ref1_stream = subject.entry_group_streams['ref1']
+      control_stream = Zip::InputStream.open( ref1_stream )
+      control_entries = described_class.entries(control_stream)
+      expect( control_entries.map{ |e| [e.name, e.get_input_stream.read]}.force ).to eq([
+        ["multiref/ref1/", ""],
+        ["multiref/ref1/datum-1", "multi-ref1-datum1\n"],
+        ["multiref/ref1/datum-2", "multi-ref1-datum2\n"]
+      ])
+    end
+    it 'e.g. ref2' do
+      ref2_stream = subject.entry_group_streams['ref2']
+      control_stream = Zip::InputStream.open( ref2_stream )
+      control_entries = described_class.entries(control_stream)
+      expect( control_entries.map{ |e| [e.name, e.get_input_stream.read]}.force ).to eq([
+        ["multiref/ref2/", ""],
+        ["multiref/ref2/datum-1", "multi-ref2-datum1\n"],
+        ["multiref/ref2/datum-2", "multi-ref2-datum2\n"]
+      ])
+    end
+  end
+
+end
diff --git a/spec/services/zip_service/zip_entry_dirs_spec.rb b/spec/services/zip_service/zip_entry_dirs_spec.rb
new file mode 100644
index 000000000..cf927855f
--- /dev/null
+++ b/spec/services/zip_service/zip_entry_dirs_spec.rb
@@ -0,0 +1,33 @@
+RSpec.describe ZipService do
+
+  let( :zip_service ){ described_class }
+
+  let( :zip_data ){ File.read zip_file }
+
+  shared_examples_for 'a correct zip entry reader' do
+    it 'gets all entries of the zip file' do
+      expect( zip_service.new(zip_data).entry_groups.keys ).to eq(expected)
+    end
+  end
+
+  context 'single entry' do
+    let( :zip_file ){ fixtures_path 'multiref.zip' }
+    let( :expected ){ %w{ref1 ref2} }
+
+    it_behaves_like 'a correct zip entry reader'
+  end
+
+  context 'more entries' do
+    let( :zip_file ){ fixtures_path 'singleref.zip' }
+    let( :expected ){ %w{ref} }
+
+    it_behaves_like 'a correct zip entry reader'
+  end
+  
+  context 'illegal file' do
+    let( :zip_file ){ fixtures_path 'nozip.zip' }
+    let( :expected ){ [] }
+
+    it_behaves_like 'a correct zip entry reader'
+  end
+end
diff --git a/spec/services/zip_service/zip_output_streams_spec.rb b/spec/services/zip_service/zip_output_streams_spec.rb
new file mode 100644
index 000000000..0b7a1e99b
--- /dev/null
+++ b/spec/services/zip_service/zip_output_streams_spec.rb
@@ -0,0 +1,17 @@
+RSpec.describe ZipService do
+  
+  subject{ described_class.new(read_fixture('multiref.zip')) }
+
+  
+  it 'can write itself to a file' do
+    streams = subject.entry_group_streams
+    streams.each do | name, stream |
+      File.write("tmp/#{name}.zip", stream.string)
+    end
+    ref1_lines = %x(unzip -l tmp/ref1.zip).split("\n").grep(%r{multiref/ref}).map(&:strip).map(&:split).map(&:last)
+    ref2_lines = %x(unzip -l tmp/ref2.zip).split("\n").grep(%r{multiref/ref}).map(&:strip).map(&:split).map(&:last)
+
+    expect( ref1_lines ).to eq %w(multiref/ref1/ multiref/ref1/datum-1 multiref/ref1/datum-2)
+    expect( ref2_lines ).to eq %w(multiref/ref2/ multiref/ref2/datum-1 multiref/ref2/datum-2)
+  end
+end
diff --git a/spec/support/fixtures_helper.rb b/spec/support/fixtures_helper.rb
new file mode 100644
index 000000000..81f6ce838
--- /dev/null
+++ b/spec/support/fixtures_helper.rb
@@ -0,0 +1,15 @@
+module Support
+  module FixturesHelper
+    def fixtures_path *segments
+      Rails.root.join( fixture_path, *segments )
+    end
+
+    def read_fixture *segments
+      File.read(fixtures_path(*segments))
+    end
+  end
+end
+
+RSpec.configure do |c|
+  c.include Support::FixturesHelper
+end
diff --git a/spec/workers/workbench_import_worker_spec.rb b/spec/workers/workbench_import_worker_spec.rb
index 9a1df5bc3..d227c7610 100644
--- a/spec/workers/workbench_import_worker_spec.rb
+++ b/spec/workers/workbench_import_worker_spec.rb
@@ -30,6 +30,8 @@ RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do
 
       worker.perform import.id
 
+  require 'pry'
+  binding.pry
       expect( File.read(worker.downloaded) ).to eq( result )
       expect( worker ).not_to be_single_entry
     end