diff options
24 files changed, 317 insertions, 28 deletions
diff --git a/app/concerns/configurable.rb b/app/concerns/configurable.rb new file mode 100644 index 000000000..c7d0f1fd9 --- /dev/null +++ b/app/concerns/configurable.rb @@ -0,0 +1,26 @@ +module Configurable + + module ClassMethods + def config &blk + blk ? blk.(configuration) : configuration + end + + private + def configuration + @__configuration__ ||= Rails::Application::Configuration.new + end + end + + module InstanceMethods + private + + def config + self.class.config + end + end + + def self.included(into) + into.extend ClassMethods + into.send :include, InstanceMethods + end +end diff --git a/app/services/file_service.rb b/app/services/file_service.rb new file mode 100644 index 000000000..efccbe24f --- /dev/null +++ b/app/services/file_service.rb @@ -0,0 +1,23 @@ +module FileService extend self + + def unique_filename( path, enum_with: with_ints ) + file_names = enum_with.map( &file_name_maker(path) ) + file_names + .drop_while( &File.method(:exists?) ) + .next + end + + def with_ints(format='%d') + (0..Float::INFINITY) + .lazy + .map{ |n| format % n } + end + + + private + + def file_name_maker path + ->(n){ [path, n].join('_') } + end + +end diff --git a/app/services/http_service.rb b/app/services/http_service.rb new file mode 100644 index 000000000..a3c4d2569 --- /dev/null +++ b/app/services/http_service.rb @@ -0,0 +1,36 @@ +class HTTPService + def self.get_resource(*args) + new.get_resource(*args) + end + + def get_resource(host:, path:, token: nil, params: {}, parse_json: false) + Faraday.new(url: host) do |c| + c.headers['Authorization'] = "Token token=#{token.inspect}" if token + c.adapter Faraday.default_adapter + + resp = c.get path, params + if resp.status == 200 + return parse_json ? JSON.parse(resp.body) : resp.body + else + raise "Error on api request status : #{resp.status} => #{resp.body}" + end + end + end + + def post_resource(host:, path:, resource_name:, token: nil, params: {}, upload: nil) + Faraday.new(url: host) do |c| + c.headers['Authorization'] = "Token token=#{token.inspect}" if token + c.request :multipart + c.request :url_encoded + c.adapter Faraday.default_adapter + + if upload + name = upload.keys.first + value, mime_type, as_name = upload.values.first + params.update( name => Faraday::UploadIO.new(value, mime_type, as_name ) ) + end + + c.post path, resource_name => params + end + end +end diff --git a/app/services/zip_service.rb b/app/services/zip_service.rb new file mode 100644 index 000000000..778bfd06d --- /dev/null +++ b/app/services/zip_service.rb @@ -0,0 +1,55 @@ +class ZipService + + attr_reader :current_entry, :zip_data + + def initialize data + @zip_data = data + @current_entry = nil + end + + class << self + def convert_entries entries + -> output_stream do + entries.each do |e| + output_stream.put_next_entry e.name + output_stream.write e.get_input_stream.read + end + end + end + + def entries input_stream + Enumerator.new do |enum| + loop{ enum << input_stream.get_next_entry } + end.lazy.take_while{ |e| e } + end + end + + def entry_groups + self.class.entries(input_stream).group_by(&method(:entry_key)) + end + + def entry_group_streams + entry_groups.map(&method(:make_stream)).to_h + end + + def entry_key entry + entry.name.split('/', -1)[-2] + end + + def make_stream pair + name, entries = pair + [name, make_stream_from( entries )] + end + + def make_stream_from entries + Zip::OutputStream.write_buffer(&self.class.convert_entries(entries)) + end + + def next_entry + @current_entry = input_stream.get_next_entry + end + + def input_stream + @__input_stream__ ||= Zip::InputStream.open(StringIO.new(zip_data)) + end +end diff --git a/app/workers/workbench_import_worker.rb b/app/workers/workbench_import_worker.rb index 18d3ae112..d1a381e0c 100644 --- a/app/workers/workbench_import_worker.rb +++ b/app/workers/workbench_import_worker.rb @@ -1,6 +1,7 @@ class WorkbenchImportWorker include Sidekiq::Worker include Rails.application.routes.url_helpers + include Configurable attr_reader :import, :downloaded @@ -17,18 +18,22 @@ class WorkbenchImportWorker path: import_path, params: {token: import.token_download}) - Tempfile.open( do | tmpfile | - tmpfile.write zipfile_data - @downloaded = tmpfile.path + path = File.join(config.dir, import.name.gsub(%r{\s+}, '-')) + unique_path = FileService.unique_filename path + Dir.mkdir unique_path + @downloaded = File.join(unique_path, import.name) + File.open(downloaded, 'wb') do | file | + file.write zipfile_data end - if one_entry? - upload(@downloaded) + if single_entry? + upload(downloaded) else split_zip.each(&method(:upload)) end end + def single_entry? true end @@ -40,6 +45,12 @@ class WorkbenchImportWorker def upload zip_file end + # Memoized Values + + def dirname + @__dirname__ ||= make_unique_dir + end + def import_host @__import_host__ ||= Rails.application.config.front_end_host end diff --git a/config/application.rb b/config/application.rb index 910ddd983..e2f6f8e7b 100644 --- a/config/application.rb +++ b/config/application.rb @@ -14,7 +14,8 @@ module ChouetteIhm # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers # -- all .rb files in that directory are automatically loaded. - config.autoload_paths << config.root.join("lib") + config.autoload_paths << config.root.join('lib') + config.autoload_paths << config.root.join('concerns') # custom exception pages config.exceptions_app = self.routes diff --git a/config/environments/development.rb b/config/environments/development.rb index 42523a761..1384d0c00 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -91,5 +91,7 @@ Rails.application.configure do # link to validation specification pages config.validation_spec = "http://www.chouette.mobi/neptune-validation/v21/" + # Local zip decompression dir + # config.i18n.available_locales = [:fr, :en] end diff --git a/config/initializers/workbench_import.rb b/config/initializers/workbench_import.rb new file mode 100644 index 000000000..1e405c9ca --- /dev/null +++ b/config/initializers/workbench_import.rb @@ -0,0 +1,5 @@ +WorkbenchImportWorker.config do | config | + config.dir = ENV.fetch('WORKBENCH_IMPORT_DIR'){ Rails.root.join 'tmp/workbench_import' } + + FileUtils.mkdir_p config.dir rescue nil +end diff --git a/hello_world b/hello_world new file mode 100644 index 000000000..3b18e512d --- /dev/null +++ b/hello_world @@ -0,0 +1 @@ +hello world diff --git a/lib/af83/http_fetcher.rb b/lib/af83/http_fetcher.rb deleted file mode 100644 index 514dd1dc9..000000000 --- a/lib/af83/http_fetcher.rb +++ /dev/null @@ -1,21 +0,0 @@ -module AF83 - class HTTPFetcher - def self.get_resource(*args) - new.get_resource(*args) - end - - def get_resource(host:, path:, token: nil, params: {}, parse_json: false) - Faraday.new(url: host) do |c| - c.headers['Authorization'] = "Token token=#{token.inspect}" if token - c.adapter Faraday.default_adapter - - resp = c.get path, params - if resp.status == 200 - return parse_json ? JSON.parse(resp.body) : resp.body - else - raise "Error on api request status : #{resp.status} => #{resp.body}" - end - end - end - end -end diff --git a/spec/concerns/configurable_spec.rb b/spec/concerns/configurable_spec.rb new file mode 100644 index 000000000..822f572c1 --- /dev/null +++ b/spec/concerns/configurable_spec.rb @@ -0,0 +1,31 @@ +RSpec.describe Configurable do + + subject do + Class.new do + include Configurable + end + end + + let( :something ){ double('something') } + + it 'can be configured' do + expect{ subject.config.anything }.to raise_error(NoMethodError) + + subject.config.something = something + + expect( subject.config.something ).to eq(something) + expect( subject.new.send(:config).something ).to eq(something) + expect( subject.new.send(:config).something ).to eq(something) + end + + it 'can be configured with a block' do + + subject.config do | c | + c.something = something + end + + expect( subject.config.something ).to eq(something) + expect( subject.new.send(:config).something ).to eq(something) + expect( subject.new.send(:config).something ).to eq(something) + end +end diff --git a/spec/fixtures/multiref.zip b/spec/fixtures/multiref.zip Binary files differindex 5b28f7cad..28ddff198 100644 --- a/spec/fixtures/multiref.zip +++ b/spec/fixtures/multiref.zip diff --git a/spec/fixtures/nozip.zip b/spec/fixtures/nozip.zip new file mode 100644 index 000000000..505bd213a --- /dev/null +++ b/spec/fixtures/nozip.zip @@ -0,0 +1 @@ +no zip file diff --git a/spec/fixtures/ref1.zip b/spec/fixtures/ref1.zip Binary files differnew file mode 100644 index 000000000..1cbd0268e --- /dev/null +++ b/spec/fixtures/ref1.zip diff --git a/spec/fixtures/ref2.zip b/spec/fixtures/ref2.zip Binary files differnew file mode 100644 index 000000000..342353b07 --- /dev/null +++ b/spec/fixtures/ref2.zip diff --git a/spec/fixtures/singleref.zip b/spec/fixtures/singleref.zip Binary files differnew file mode 100644 index 000000000..4aee23614 --- /dev/null +++ b/spec/fixtures/singleref.zip diff --git a/spec/fixtures/uniref.zip b/spec/fixtures/uniref.zip Binary files differdeleted file mode 100644 index 86b688b51..000000000 --- a/spec/fixtures/uniref.zip +++ /dev/null diff --git a/spec/requests/api/v1/netex_import_spec.rb b/spec/requests/api/v1/netex_import_spec.rb index e67cef9a2..1c34d5544 100644 --- a/spec/requests/api/v1/netex_import_spec.rb +++ b/spec/requests/api/v1/netex_import_spec.rb @@ -4,7 +4,7 @@ RSpec.describe "NetexImport", type: :request do let( :referential ){ create :referential } - let( :file_path ){'spec/fixtures/neptune.zip'} + let( :file_path ){ fixtures_path 'singleref.zip' } let( :file ){ fixture_file_upload( file_path ) } let( :post_request ) do diff --git a/spec/services/file_service_spec.rb b/spec/services/file_service_spec.rb new file mode 100644 index 000000000..90e164408 --- /dev/null +++ b/spec/services/file_service_spec.rb @@ -0,0 +1,16 @@ +RSpec.describe FileService do + + it 'computes a unique filename' do + expect( File ).to receive(:exists?).with('xxx/yyy_0').and_return( false ) + + expect(described_class.unique_filename('xxx/yyy')).to eq('xxx/yyy_0') + end + + it 'handles duplicate names by means of a counter' do + expect( File ).to receive(:exists?).with('xxx/yyy_0').and_return( true ) + expect( File ).to receive(:exists?).with('xxx/yyy_1').and_return( true ) + expect( File ).to receive(:exists?).with('xxx/yyy_2').and_return( false ) + + expect(described_class.unique_filename('xxx/yyy')).to eq('xxx/yyy_2') + end +end diff --git a/spec/services/zip_service/zip_entry_data_spec.rb b/spec/services/zip_service/zip_entry_data_spec.rb new file mode 100644 index 000000000..6bfaa8cc4 --- /dev/null +++ b/spec/services/zip_service/zip_entry_data_spec.rb @@ -0,0 +1,35 @@ +RSpec.describe ZipService do + + subject{ described_class.new(read_fixture('multiref.zip')) } + + let( :ref1_zipdata ){ read_fixture('ref1.zip') } + let( :ref2_zipdata ){ read_fixture('ref2.zip') } + + it 'can group all entries' do + expect( subject.entry_groups.keys ).to eq(%w{ref1 ref2}) + end + + context 'creates correct zip data for each subdir' do + it 'e.g. ref1' do + ref1_stream = subject.entry_group_streams['ref1'] + control_stream = Zip::InputStream.open( ref1_stream ) + control_entries = described_class.entries(control_stream) + expect( control_entries.map{ |e| [e.name, e.get_input_stream.read]}.force ).to eq([ + ["multiref/ref1/", ""], + ["multiref/ref1/datum-1", "multi-ref1-datum1\n"], + ["multiref/ref1/datum-2", "multi-ref1-datum2\n"] + ]) + end + it 'e.g. ref2' do + ref2_stream = subject.entry_group_streams['ref2'] + control_stream = Zip::InputStream.open( ref2_stream ) + control_entries = described_class.entries(control_stream) + expect( control_entries.map{ |e| [e.name, e.get_input_stream.read]}.force ).to eq([ + ["multiref/ref2/", ""], + ["multiref/ref2/datum-1", "multi-ref2-datum1\n"], + ["multiref/ref2/datum-2", "multi-ref2-datum2\n"] + ]) + end + end + +end diff --git a/spec/services/zip_service/zip_entry_dirs_spec.rb b/spec/services/zip_service/zip_entry_dirs_spec.rb new file mode 100644 index 000000000..cf927855f --- /dev/null +++ b/spec/services/zip_service/zip_entry_dirs_spec.rb @@ -0,0 +1,33 @@ +RSpec.describe ZipService do + + let( :zip_service ){ described_class } + + let( :zip_data ){ File.read zip_file } + + shared_examples_for 'a correct zip entry reader' do + it 'gets all entries of the zip file' do + expect( zip_service.new(zip_data).entry_groups.keys ).to eq(expected) + end + end + + context 'single entry' do + let( :zip_file ){ fixtures_path 'multiref.zip' } + let( :expected ){ %w{ref1 ref2} } + + it_behaves_like 'a correct zip entry reader' + end + + context 'more entries' do + let( :zip_file ){ fixtures_path 'singleref.zip' } + let( :expected ){ %w{ref} } + + it_behaves_like 'a correct zip entry reader' + end + + context 'illegal file' do + let( :zip_file ){ fixtures_path 'nozip.zip' } + let( :expected ){ [] } + + it_behaves_like 'a correct zip entry reader' + end +end diff --git a/spec/services/zip_service/zip_output_streams_spec.rb b/spec/services/zip_service/zip_output_streams_spec.rb new file mode 100644 index 000000000..0b7a1e99b --- /dev/null +++ b/spec/services/zip_service/zip_output_streams_spec.rb @@ -0,0 +1,17 @@ +RSpec.describe ZipService do + + subject{ described_class.new(read_fixture('multiref.zip')) } + + + it 'can write itself to a file' do + streams = subject.entry_group_streams + streams.each do | name, stream | + File.write("tmp/#{name}.zip", stream.string) + end + ref1_lines = %x(unzip -l tmp/ref1.zip).split("\n").grep(%r{multiref/ref}).map(&:strip).map(&:split).map(&:last) + ref2_lines = %x(unzip -l tmp/ref2.zip).split("\n").grep(%r{multiref/ref}).map(&:strip).map(&:split).map(&:last) + + expect( ref1_lines ).to eq %w(multiref/ref1/ multiref/ref1/datum-1 multiref/ref1/datum-2) + expect( ref2_lines ).to eq %w(multiref/ref2/ multiref/ref2/datum-1 multiref/ref2/datum-2) + end +end diff --git a/spec/support/fixtures_helper.rb b/spec/support/fixtures_helper.rb new file mode 100644 index 000000000..81f6ce838 --- /dev/null +++ b/spec/support/fixtures_helper.rb @@ -0,0 +1,15 @@ +module Support + module FixturesHelper + def fixtures_path *segments + Rails.root.join( fixture_path, *segments ) + end + + def read_fixture *segments + File.read(fixtures_path(*segments)) + end + end +end + +RSpec.configure do |c| + c.include Support::FixturesHelper +end diff --git a/spec/workers/workbench_import_worker_spec.rb b/spec/workers/workbench_import_worker_spec.rb index 9a1df5bc3..d227c7610 100644 --- a/spec/workers/workbench_import_worker_spec.rb +++ b/spec/workers/workbench_import_worker_spec.rb @@ -30,6 +30,8 @@ RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do worker.perform import.id + require 'pry' + binding.pry expect( File.read(worker.downloaded) ).to eq( result ) expect( worker ).not_to be_single_entry end |