diff options
Diffstat (limited to 'app')
| -rw-r--r-- | app/models/organisation.rb | 63 | ||||
| -rw-r--r-- | app/services/zip_service.rb | 29 | ||||
| -rw-r--r-- | app/workers/workbench_import_worker.rb | 92 | ||||
| -rw-r--r-- | app/workers/workbench_import_worker/object_state_updater.rb | 36 |
4 files changed, 146 insertions, 74 deletions
diff --git a/app/models/organisation.rb b/app/models/organisation.rb index f6fba2d67..4343c87af 100644 --- a/app/models/organisation.rb +++ b/app/models/organisation.rb @@ -18,36 +18,39 @@ class Organisation < ActiveRecord::Base validates_presence_of :name validates_uniqueness_of :code - def self.portail_api_request - conf = Rails.application.config.try(:stif_portail_api) - raise 'Rails.application.config.stif_portail_api configuration is not defined' unless conf - - HTTPService.get_json_resource( - host: conf[:url], - path: '/api/v1/organizations', - token: conf[:key]) - end + class << self + + def portail_api_request + conf = Rails.application.config.try(:stif_portail_api) + raise 'Rails.application.config.stif_portail_api configuration is not defined' unless conf - def self.sync_update code, name, scope - org = Organisation.find_or_initialize_by(code: code) - if scope - org.sso_attributes ||= {} - if org.sso_attributes['functional_scope'] != scope - org.sso_attributes['functional_scope'] = scope - # FIXME see #1941 - org.sso_attributes_will_change! + HTTPService.get_json_resource( + host: conf[:url], + path: '/api/v1/organizations', + token: conf[:key]) + end + + def sync_update code, name, scope + org = Organisation.find_or_initialize_by(code: code) + if scope + org.sso_attributes ||= {} + if org.sso_attributes['functional_scope'] != scope + org.sso_attributes['functional_scope'] = scope + # FIXME see #1941 + org.sso_attributes_will_change! + end end + org.name = name + org.synced_at = Time.now + org.save + org end - org.name = name - org.synced_at = Time.now - org.save - org - end - def self.portail_sync - self.portail_api_request.each do |el| - org = self.sync_update el['code'], el['name'], el['functional_scope'] - puts "✓ Organisation #{org.name} has been updated" unless Rails.env.test? + def portail_sync + portail_api_request.each do |el| + org = self.sync_update el['code'], el['name'], el['functional_scope'] + puts "✓ Organisation #{org.name} has been updated" unless Rails.env.test? + end end end @@ -64,4 +67,12 @@ class Organisation < ActiveRecord::Base raise ActiveRecord::RecordNotFound end + def functional_scope + JSON.parse( (sso_attributes || {}).fetch('functional_scope', '[]') ) + end + + def lines_set + STIF::CodifligneLineId.lines_set_from_functional_scope( functional_scope ) + end + end diff --git a/app/services/zip_service.rb b/app/services/zip_service.rb index 7a4bdad1b..a34dd97cc 100644 --- a/app/services/zip_service.rb +++ b/app/services/zip_service.rb @@ -1,12 +1,16 @@ class ZipService - class Subdir < Struct.new(:name, :stream, :spurious) + class Subdir < Struct.new(:name, :stream, :spurious, :foreign_lines) + def ok? + foreign_lines.empty? && spurious.empty? + end end - attr_reader :current_key, :current_output, :current_spurious, :yielder + attr_reader :allowed_lines, :current_key, :foreign_lines, :current_output, :current_spurious, :yielder - def initialize data + def initialize data, allowed_lines @zip_data = StringIO.new(data) + @allowed_lines = allowed_lines @current_key = nil @current_output = nil end @@ -35,7 +39,8 @@ class ZipService end def add_to_current_output entry - return if is_spurious! entry.name + return if is_spurious!(entry.name) || is_foreign_line!(entry.name) + current_output.put_next_entry entry.name write_to_current_output entry.get_input_stream end @@ -52,20 +57,22 @@ class ZipService current_key, # Second part of the solution, yield the closed stream current_output.close_buffer, - current_spurious) + current_spurious, + foreign_lines) end end def open_new_output entry_key @current_key = entry_key # First piece of the solution, use internal way to create a Zip::OutputStream - @current_output = Zip::OutputStream.new(StringIO.new(''), true, nil) + @current_output = Zip::OutputStream.new(StringIO.new(''), true, nil) @current_spurious = [] + @foreign_lines = [] end def entry_key entry # last dir name File.dirname.split("/").last - entry.name.split('/', -1)[-2] + entry.name.split('/').first end def is_spurious! entry_name @@ -75,4 +82,12 @@ class ZipService current_spurious << segments.second return true end + + def is_foreign_line! entry_name + STIF::NetexFile::Frame.get_line_object_id(entry_name).tap do | line_object_id | + return nil unless line_object_id + return nil if line_object_id.in? allowed_lines + foreign_lines << line_object_id + end + end end diff --git a/app/workers/workbench_import_worker.rb b/app/workers/workbench_import_worker.rb index de51efded..6420be835 100644 --- a/app/workers/workbench_import_worker.rb +++ b/app/workers/workbench_import_worker.rb @@ -3,29 +3,25 @@ class WorkbenchImportWorker include Rails.application.routes.url_helpers include Configurable + include ObjectStateUpdater + + attr_reader :entries, :workbench_import + # Workers # ======= def perform(import_id) - @workbench_import = WorkbenchImport.find(import_id) - @response = nil - @workbench_import.update(status: 'running', started_at: Time.now) - downloaded = download - zip_service = ZipService.new(downloaded) + @entries = 0 + @workbench_import ||= WorkbenchImport.find(import_id) + + workbench_import.update(status: 'running', started_at: Time.now) + zip_service = ZipService.new(downloaded, allowed_lines) upload zip_service - @workbench_import.update(ended_at: Time.now) + workbench_import.update(ended_at: Time.now) rescue Zip::Error handle_corrupt_zip_file end - def download - logger.info "HTTP GET #{import_url}" - HTTPService.get_resource( - host: import_host, - path: import_path, - params: {token: @workbench_import.token_download}).body - end - def execute_post eg_name, eg_file logger.info "HTTP POST #{export_url} (for #{complete_entry_group_name(eg_name)})" HTTPService.post_resource( @@ -35,48 +31,43 @@ class WorkbenchImportWorker end def handle_corrupt_zip_file - @workbench_import.messages.create(criticity: :error, message_key: 'corrupt_zip_file', message_attributes: {source_filename: @workbench_import.file.file.file}) + workbench_import.messages.create(criticity: :error, message_key: 'corrupt_zip_file', message_attributes: {source_filename: workbench_import.file.file.file}) end def upload zip_service entry_group_streams = zip_service.subdirs - @workbench_import.update total_steps: entry_group_streams.size entry_group_streams.each_with_index(&method(:upload_entry_group)) + workbench_import.update total_steps: @entries rescue Exception => e logger.error e.message - @workbench_import.update( current_step: entry_group_streams.size, status: 'failed' ) + workbench_import.update( current_step: @entries, status: 'failed' ) raise end - def update_object_state entry, count - @workbench_import.update( current_step: count ) - unless entry.spurious.empty? - @workbench_import.messages.create( - criticity: :warning, - message_key: 'inconsistent_zip_file', - message_attributes: { - 'source_filename' => @workbench_import.file.file.file, - 'spurious_dirs' => entry.spurious.join(', ') - }) - end - end def upload_entry_group entry, element_count update_object_state entry, element_count.succ + return unless entry.ok? # status = retry_service.execute(&upload_entry_group_proc(entry)) - eg_name = entry.name - eg_stream = entry.stream + upload_entry_group_stream entry.name, entry.stream + end + def upload_entry_group_stream eg_name, eg_stream FileUtils.mkdir_p(Rails.root.join('tmp', 'imports')) - eg_file = File.new(Rails.root.join('tmp', 'imports', "WorkbenchImport_#{eg_name}_#{$$}.zip"), 'wb').tap do |file| + File.open(Rails.root.join('tmp', 'imports', "WorkbenchImport_#{eg_name}_#{$$}.zip"), 'wb') do |file| eg_stream.rewind file.write eg_stream.read end - eg_file.close - eg_file = File.new(Rails.root.join('tmp', 'imports', "WorkbenchImport_#{eg_name}_#{$$}.zip")) + + upload_entry_group_tmpfile eg_name, File.new(Rails.root.join('tmp', 'imports', "WorkbenchImport_#{eg_name}_#{$$}.zip")) + end + + def upload_entry_group_tmpfile eg_name, eg_file result = execute_post eg_name, eg_file if result && result.status < 400 + @entries += 1 + workbench_import.update( current_step: @entries ) result else raise StopIteration, result.body @@ -91,7 +82,7 @@ class WorkbenchImportWorker # ======= def complete_entry_group_name entry_group_name - [@workbench_import.name, entry_group_name].join("--") + [workbench_import.name, entry_group_name].join("--") end # Constants @@ -111,7 +102,7 @@ class WorkbenchImportWorker Rails.application.config.rails_host end def import_path - @__import_path__ ||= download_workbench_import_path(@workbench_import.workbench, @workbench_import) + @__import_path__ ||= download_workbench_import_path(workbench_import.workbench, workbench_import) end def import_url @__import_url__ ||= File.join(import_host, import_path) @@ -119,10 +110,29 @@ class WorkbenchImportWorker def params file, name { netex_import: - { parent_id: @workbench_import.id, - parent_type: @workbench_import.class.name, - workbench_id: @workbench_import.workbench_id, - name: name, - file: HTTPService.upload(file, 'application/zip', "#{name}.zip") } } + { parent_id: workbench_import.id, + parent_type: workbench_import.class.name, + workbench_id: workbench_import.workbench_id, + name: name, + file: HTTPService.upload(file, 'application/zip', "#{name}.zip") } } + end + + # Lazy Values + # =========== + + def allowed_lines + @__allowed_lines__ ||= workbench_import.workbench.organisation.lines_set end + def downloaded + @__downloaded__ ||= download_response.body + end + def download_response + @__download_response__ ||= HTTPService.get_resource( + host: import_host, + path: import_path, + params: {token: workbench_import.token_download}).tap do + logger.info "HTTP GET #{import_url}" + end + end + end diff --git a/app/workers/workbench_import_worker/object_state_updater.rb b/app/workers/workbench_import_worker/object_state_updater.rb new file mode 100644 index 000000000..e9cc081b7 --- /dev/null +++ b/app/workers/workbench_import_worker/object_state_updater.rb @@ -0,0 +1,36 @@ + +class WorkbenchImportWorker + module ObjectStateUpdater + + def update_object_state entry, count + workbench_import.update( total_steps: count ) + update_spurious entry + update_foreign_lines entry + end + + + private + + def update_foreign_lines entry + return if entry.foreign_lines.empty? + workbench_import.messages.create( + criticity: :warning, + message_key: 'foreign_lines_in_referential', + message_attributes: { + 'source_filename' => workbench_import.file.file.file, + 'foreign_lines' => entry.foreign_lines.join(', ') + }) + end + + def update_spurious entry + return if entry.spurious.empty? + workbench_import.messages.create( + criticity: :warning, + message_key: 'inconsistent_zip_file', + message_attributes: { + 'source_filename' => workbench_import.file.file.file, + 'spurious_dirs' => entry.spurious.join(', ') + }) + end + end +end |