Merge pull request #124 from af83/5006-wb_import_filter_refs_with_foreign_lines

5006 wb import filter refs with foreign lines

4 files changed, 146 insertions, 74 deletions

diff --git a/app/models/organisation.rb b/app/models/organisation.rb
index f6fba2d67..4343c87af 100644
--- a/app/models/organisation.rb
+++ b/app/models/organisation.rb
@@ -18,36 +18,39 @@ class Organisation < ActiveRecord::Base
   validates_presence_of :name
   validates_uniqueness_of :code
 
-  def self.portail_api_request
-    conf = Rails.application.config.try(:stif_portail_api)
-    raise 'Rails.application.config.stif_portail_api configuration is not defined' unless conf
-
-    HTTPService.get_json_resource(
-      host: conf[:url],
-      path: '/api/v1/organizations',
-      token: conf[:key])
-  end
+  class << self
+
+    def portail_api_request
+      conf = Rails.application.config.try(:stif_portail_api)
+      raise 'Rails.application.config.stif_portail_api configuration is not defined' unless conf
 
-  def self.sync_update code, name, scope
-    org = Organisation.find_or_initialize_by(code: code)
-    if scope
-      org.sso_attributes ||= {}
-      if org.sso_attributes['functional_scope'] != scope
-        org.sso_attributes['functional_scope'] = scope
-        # FIXME see #1941
-        org.sso_attributes_will_change!
+      HTTPService.get_json_resource(
+        host: conf[:url],
+        path: '/api/v1/organizations',
+        token: conf[:key])
+    end
+
+    def sync_update code, name, scope
+      org = Organisation.find_or_initialize_by(code: code)
+      if scope
+        org.sso_attributes ||= {}
+        if org.sso_attributes['functional_scope'] != scope
+          org.sso_attributes['functional_scope'] = scope
+          # FIXME see #1941
+          org.sso_attributes_will_change!
+        end
       end
+      org.name      = name
+      org.synced_at = Time.now
+      org.save
+      org
     end
-    org.name      = name
-    org.synced_at = Time.now
-    org.save
-    org
-  end
 
-  def self.portail_sync
-    self.portail_api_request.each do |el|
-      org = self.sync_update el['code'], el['name'], el['functional_scope']
-      puts "✓ Organisation #{org.name} has been updated" unless Rails.env.test?
+    def portail_sync
+      portail_api_request.each do |el|
+        org = self.sync_update el['code'], el['name'], el['functional_scope']
+        puts "✓ Organisation #{org.name} has been updated" unless Rails.env.test?
+      end
     end
   end
 
@@ -64,4 +67,12 @@ class Organisation < ActiveRecord::Base
     raise ActiveRecord::RecordNotFound
   end
 
+  def functional_scope
+    JSON.parse( (sso_attributes || {}).fetch('functional_scope', '[]') )
+  end
+
+  def lines_set
+    STIF::CodifligneLineId.lines_set_from_functional_scope( functional_scope )
+  end
+
 end
diff --git a/app/services/zip_service.rb b/app/services/zip_service.rb
index 7a4bdad1b..a34dd97cc 100644
--- a/app/services/zip_service.rb
+++ b/app/services/zip_service.rb
@@ -1,12 +1,16 @@
 class ZipService
 
-  class Subdir < Struct.new(:name, :stream, :spurious)
+  class Subdir < Struct.new(:name, :stream, :spurious, :foreign_lines)
+    def ok?
+      foreign_lines.empty? && spurious.empty?
+    end
   end
 
-  attr_reader :current_key, :current_output, :current_spurious, :yielder
+  attr_reader :allowed_lines, :current_key, :foreign_lines, :current_output, :current_spurious, :yielder
 
-  def initialize data
+  def initialize data, allowed_lines
     @zip_data       = StringIO.new(data)
+    @allowed_lines  = allowed_lines
     @current_key    = nil
     @current_output = nil
   end
@@ -35,7 +39,8 @@ class ZipService
   end
 
   def add_to_current_output entry
-    return if is_spurious! entry.name
+    return if is_spurious!(entry.name) || is_foreign_line!(entry.name)
+
     current_output.put_next_entry entry.name
     write_to_current_output entry.get_input_stream
   end
@@ -52,20 +57,22 @@ class ZipService
         current_key,
         # Second part of the solution, yield the closed stream
         current_output.close_buffer,
-        current_spurious)
+        current_spurious,
+        foreign_lines)
     end
   end
 
   def open_new_output entry_key
     @current_key    = entry_key
     # First piece of the solution, use internal way to create a Zip::OutputStream
-    @current_output = Zip::OutputStream.new(StringIO.new(''), true, nil)
+    @current_output   = Zip::OutputStream.new(StringIO.new(''), true, nil)
     @current_spurious = []
+    @foreign_lines    = []
   end
 
   def entry_key entry
     # last dir name File.dirname.split("/").last
-    entry.name.split('/', -1)[-2]
+    entry.name.split('/').first
   end
 
   def is_spurious! entry_name
@@ -75,4 +82,12 @@ class ZipService
     current_spurious << segments.second
     return true
   end
+
+  def is_foreign_line! entry_name
+    STIF::NetexFile::Frame.get_line_object_id(entry_name).tap do | line_object_id |
+      return nil unless line_object_id
+      return nil if line_object_id.in? allowed_lines
+      foreign_lines << line_object_id
+    end
+  end
 end
diff --git a/app/workers/workbench_import_worker.rb b/app/workers/workbench_import_worker.rb
index de51efded..6420be835 100644
--- a/app/workers/workbench_import_worker.rb
+++ b/app/workers/workbench_import_worker.rb
@@ -3,29 +3,25 @@ class WorkbenchImportWorker
   include Rails.application.routes.url_helpers
   include Configurable
 
+  include ObjectStateUpdater
+
+  attr_reader :entries, :workbench_import
+
   # Workers
   # =======
 
   def perform(import_id)
-    @workbench_import = WorkbenchImport.find(import_id)
-    @response         = nil
-    @workbench_import.update(status: 'running', started_at: Time.now)
-    downloaded  = download
-    zip_service = ZipService.new(downloaded)
+    @entries = 0
+    @workbench_import ||= WorkbenchImport.find(import_id)
+
+    workbench_import.update(status: 'running', started_at: Time.now)
+    zip_service = ZipService.new(downloaded, allowed_lines)
     upload zip_service
-    @workbench_import.update(ended_at: Time.now)
+    workbench_import.update(ended_at: Time.now)
   rescue Zip::Error
     handle_corrupt_zip_file
   end
 
-  def download
-    logger.info  "HTTP GET #{import_url}"
-    HTTPService.get_resource(
-      host: import_host,
-      path: import_path,
-      params: {token: @workbench_import.token_download}).body
-  end
-
   def execute_post eg_name, eg_file
     logger.info  "HTTP POST #{export_url} (for #{complete_entry_group_name(eg_name)})"
     HTTPService.post_resource(
@@ -35,48 +31,43 @@ class WorkbenchImportWorker
   end
 
   def handle_corrupt_zip_file
-    @workbench_import.messages.create(criticity: :error, message_key: 'corrupt_zip_file', message_attributes: {source_filename: @workbench_import.file.file.file})
+    workbench_import.messages.create(criticity: :error, message_key: 'corrupt_zip_file', message_attributes: {source_filename: workbench_import.file.file.file})
   end
 
   def upload zip_service
     entry_group_streams = zip_service.subdirs
-    @workbench_import.update total_steps: entry_group_streams.size
     entry_group_streams.each_with_index(&method(:upload_entry_group))
+    workbench_import.update total_steps: @entries
   rescue Exception => e
     logger.error e.message
-    @workbench_import.update( current_step: entry_group_streams.size, status: 'failed' )
+    workbench_import.update( current_step: @entries, status: 'failed' )
     raise
   end
 
-  def update_object_state entry, count
-    @workbench_import.update( current_step: count )
-    unless entry.spurious.empty?
-      @workbench_import.messages.create(
-        criticity: :warning,
-        message_key: 'inconsistent_zip_file',
-        message_attributes: {
-          'source_filename' => @workbench_import.file.file.file,
-          'spurious_dirs'   => entry.spurious.join(', ')
-        }) 
-    end
-  end
 
   def upload_entry_group entry, element_count
     update_object_state entry, element_count.succ
+    return unless entry.ok?
     # status = retry_service.execute(&upload_entry_group_proc(entry))
-    eg_name = entry.name
-    eg_stream = entry.stream
+    upload_entry_group_stream entry.name, entry.stream
+  end
 
+  def upload_entry_group_stream eg_name, eg_stream
     FileUtils.mkdir_p(Rails.root.join('tmp', 'imports'))
 
-    eg_file = File.new(Rails.root.join('tmp', 'imports', "WorkbenchImport_#{eg_name}_#{$$}.zip"), 'wb').tap do |file|
+    File.open(Rails.root.join('tmp', 'imports', "WorkbenchImport_#{eg_name}_#{$$}.zip"), 'wb') do |file|
       eg_stream.rewind
       file.write eg_stream.read
     end
-    eg_file.close
-    eg_file = File.new(Rails.root.join('tmp', 'imports', "WorkbenchImport_#{eg_name}_#{$$}.zip"))
+
+    upload_entry_group_tmpfile eg_name, File.new(Rails.root.join('tmp', 'imports', "WorkbenchImport_#{eg_name}_#{$$}.zip"))
+  end
+    
+  def upload_entry_group_tmpfile eg_name, eg_file
     result = execute_post eg_name, eg_file
     if result && result.status < 400
+      @entries += 1
+      workbench_import.update( current_step: @entries )
       result
     else
       raise StopIteration, result.body
@@ -91,7 +82,7 @@ class WorkbenchImportWorker
   # =======
 
   def complete_entry_group_name entry_group_name
-    [@workbench_import.name, entry_group_name].join("--")
+    [workbench_import.name, entry_group_name].join("--")
   end
 
   # Constants
@@ -111,7 +102,7 @@ class WorkbenchImportWorker
     Rails.application.config.rails_host
   end
   def import_path
-    @__import_path__ ||= download_workbench_import_path(@workbench_import.workbench, @workbench_import)
+    @__import_path__ ||= download_workbench_import_path(workbench_import.workbench, workbench_import)
   end
   def import_url
     @__import_url__ ||= File.join(import_host, import_path)
@@ -119,10 +110,29 @@ class WorkbenchImportWorker
 
   def params file, name
     { netex_import:
-        { parent_id: @workbench_import.id,
-          parent_type: @workbench_import.class.name,
-          workbench_id: @workbench_import.workbench_id,
-          name: name,
-          file: HTTPService.upload(file, 'application/zip', "#{name}.zip") } }
+      { parent_id: workbench_import.id,
+        parent_type: workbench_import.class.name,
+        workbench_id: workbench_import.workbench_id,
+        name: name,
+        file: HTTPService.upload(file, 'application/zip', "#{name}.zip") } }
+  end
+
+  # Lazy Values
+  # ===========
+
+  def allowed_lines
+    @__allowed_lines__ ||= workbench_import.workbench.organisation.lines_set
   end
+  def downloaded
+    @__downloaded__ ||= download_response.body
+  end
+  def download_response
+    @__download_response__ ||= HTTPService.get_resource(
+      host: import_host,
+      path: import_path,
+      params: {token: workbench_import.token_download}).tap do
+        logger.info  "HTTP GET #{import_url}"
+      end
+  end
+
 end
diff --git a/app/workers/workbench_import_worker/object_state_updater.rb b/app/workers/workbench_import_worker/object_state_updater.rb
new file mode 100644
index 000000000..e9cc081b7
--- /dev/null
+++ b/app/workers/workbench_import_worker/object_state_updater.rb
@@ -0,0 +1,36 @@
+
+class WorkbenchImportWorker
+  module ObjectStateUpdater
+
+    def update_object_state entry, count
+      workbench_import.update( total_steps: count )
+      update_spurious entry
+      update_foreign_lines entry
+    end
+
+
+    private
+
+    def update_foreign_lines entry
+      return if entry.foreign_lines.empty?
+      workbench_import.messages.create(
+        criticity: :warning,
+        message_key: 'foreign_lines_in_referential',
+        message_attributes: {
+          'source_filename' => workbench_import.file.file.file,
+          'foreign_lines'   => entry.foreign_lines.join(', ')
+        }) 
+    end
+
+    def update_spurious entry
+      return if entry.spurious.empty?
+      workbench_import.messages.create(
+        criticity: :warning,
+        message_key: 'inconsistent_zip_file',
+        message_attributes: {
+          'source_filename' => workbench_import.file.file.file,
+          'spurious_dirs'   => entry.spurious.join(', ')
+        }) 
+    end
+  end
+end