Rework imports to use Resources and make the a little more verbose

15 files changed, 276 insertions, 74 deletions

diff --git a/app/assets/stylesheets/application.sass b/app/assets/stylesheets/application.sass
index 3f8467efe..632ade179 100644
--- a/app/assets/stylesheets/application.sass
+++ b/app/assets/stylesheets/application.sass
@@ -21,3 +21,6 @@
 @import 'modules/import_messages'
 
 @import 'flag-icon'
+
+span.fa + span
+  margin-left: 0.2em
diff --git a/app/controllers/import_resources_controller.rb b/app/controllers/import_resources_controller.rb
index 1535fd171..46f8f0337 100644
--- a/app/controllers/import_resources_controller.rb
+++ b/app/controllers/import_resources_controller.rb
@@ -24,6 +24,15 @@ class ImportResourcesController < ChouetteController
     @import_resources ||= parent.resources
   end
 
+  def resource
+    @import ||= Import::Base.find params[:import_id]
+    @import_resource ||= begin
+      import_resource = Import::Resource.find params[:id]
+      raise ActiveRecord::RecordNotFound unless import_resource.import == @import
+      import_resource
+    end
+  end
+
   private
 
   def decorate_import_resources(import_resources)
diff --git a/app/helpers/exports_helper.rb b/app/helpers/exports_helper.rb
index 2e784ad35..f30a80ed9 100644
--- a/app/helpers/exports_helper.rb
+++ b/app/helpers/exports_helper.rb
@@ -17,7 +17,7 @@ module ExportsHelper
       message.message_attributes["text"]
     else
       t([message.class.name.underscore.gsub('/', '_').pluralize, message.message_key].join('.'), message.message_attributes&.symbolize_keys || {})
-    end
+    end.html_safe
   end
 
   def fields_for_export_task_format(form)
diff --git a/app/helpers/imports_helper.rb b/app/helpers/imports_helper.rb
index 140660153..a297c2521 100644
--- a/app/helpers/imports_helper.rb
+++ b/app/helpers/imports_helper.rb
@@ -2,33 +2,47 @@
 module ImportsHelper
 
   # Import statuses helper
-  def import_status(status)
-    if %w[new running pending].include? status
+  def import_status(status, verbose: false)
+    status = status.to_s.downcase
+    out = if %w[new running pending].include? status
       content_tag :span, '', class: "fa fa-clock-o"
     else
       cls =''
       cls = 'success' if status == 'successful'
+      cls = 'success' if status == 'ok'
       cls = 'warning' if status == 'warning'
-      cls = 'danger' if %w[failed aborted canceled].include? status
+      cls = 'danger' if %w[failed aborted canceled error].include? status
 
       content_tag :span, '', class: "fa fa-circle text-#{cls}"
     end
+    if verbose
+      out += content_tag :span do
+        txt = "imports.status.#{status}".t(fallback: "")
+      end
+    end
+    out
   end
 
   # Compliance check set messages
   def bootstrap_class_for_message_criticity message_criticity
-    case message_criticity
-      when "error"
+    case message_criticity.downcase
+      when "error", "aborted"
         "alert alert-danger"
       when "warning"
         "alert alert-warning"
       when "info"
         "alert alert-info"
+      when "ok", "success"
+        "alert alert-success"
       else
         message_criticity.to_s
     end
   end
 
+  def import_message_content message
+    export_message_content message
+  end
+
   ##############################
   #      TO CLEAN!!!
   ##############################
diff --git a/app/helpers/table_builder_helper.rb b/app/helpers/table_builder_helper.rb
index e2aa2e9ea..8c73cb33c 100644
--- a/app/helpers/table_builder_helper.rb
+++ b/app/helpers/table_builder_helper.rb
@@ -153,7 +153,7 @@ module TableBuilderHelper
             i = columns.index(column)
 
             if overhead[i].blank?
-              if (i > 0) && (overhead[i - 1][:width] > 1)
+              if (i > 0) && overhead[i - 1][:width] && (overhead[i - 1][:width] > 1)
                 clsArrayH = overhead[i - 1][:cls].split
 
                 hcont << content_tag(:th, build_column_header(
@@ -225,7 +225,7 @@ module TableBuilderHelper
 
         if column.linkable?
           path = column.link_to(item)
-          link = value.present? && path.present? ? link_to(value, path) : ""
+          link = value.present? && path.present? ? link_to(value, path) : value
 
           if overhead.empty?
             bcont << content_tag(:td, link, title: 'Voir', class: extra_class)
@@ -234,7 +234,7 @@ module TableBuilderHelper
             i = columns.index(column)
 
             if overhead[i].blank?
-              if (i > 0) && (overhead[i - 1][:width] > 1)
+              if (i > 0) && overhead[i - 1][:width] && (overhead[i - 1][:width] > 1)
                 clsArrayAlt = overhead[i - 1][:cls].split
 
                 bcont << content_tag(:td, link, title: 'Voir', class: td_cls(clsArrayAlt, extra_class))
diff --git a/app/models/concerns/iev_interfaces/resource.rb b/app/models/concerns/iev_interfaces/resource.rb
index 7f8c3eefd..254f88a33 100644
--- a/app/models/concerns/iev_interfaces/resource.rb
+++ b/app/models/concerns/iev_interfaces/resource.rb
@@ -4,6 +4,24 @@ module IevInterfaces::Resource
   included do
     extend Enumerize
     enumerize :status, in: %i(OK ERROR WARNING IGNORED), scope: true
-    validates_presence_of :name, :resource_type, :reference
+    validates_presence_of :name, :resource_type
+  end
+
+  def update_status_from_importer importer_status
+    self.update status: status_from_importer(importer_status)
+  end
+
+  def status_from_importer importer_status
+    return nil unless importer_status.present?
+    {
+      new: nil,
+      pending: nil,
+      successful: :OK,
+      warning: :WARNING,
+      failed: :ERROR,
+      running: nil,
+      aborted: :ERROR,
+      canceled: :ERROR
+    }[importer_status.to_sym]
   end
 end
diff --git a/app/models/import/base.rb b/app/models/import/base.rb
index f98e359d4..333fb0f56 100644
--- a/app/models/import/base.rb
+++ b/app/models/import/base.rb
@@ -39,6 +39,14 @@ class Import::Base < ApplicationModel
 
   private
 
+  def failed!
+    update status: :failed
+  end
+
+  def aborted!
+    update status: :aborted
+  end
+
   def initialize_fields
     super
     self.token_download ||= SecureRandom.urlsafe_base64
diff --git a/app/models/import/gtfs.rb b/app/models/import/gtfs.rb
index ceb849bd8..e1bac36a6 100644
--- a/app/models/import/gtfs.rb
+++ b/app/models/import/gtfs.rb
@@ -1,10 +1,28 @@
 class Import::Gtfs < Import::Base
   after_commit :launch_worker, :on => :create
 
+  after_commit do
+    main_resource.update_status_from_importer self.status
+    true
+  end
+
   def launch_worker
     GtfsImportWorker.perform_async id
   end
 
+  def main_resource
+    @resource ||= parent.resources.find_or_create_by(name: self.name, resource_type: "referential", reference: self.name)
+  end
+
+  def notify_parent
+    super
+    main_resource.update_status_from_importer self.status
+  end
+
+  def create_message args
+    main_resource.messages.create args
+  end
+
   def import
     update status: 'running', started_at: Time.now
 
@@ -35,6 +53,7 @@ class Import::Gtfs < Import::Base
       workbench_id: workbench.id,
       metadatas: [referential_metadata]
     )
+    main_resource.update referential: referential
   end
 
   def referential_metadata
@@ -131,17 +150,21 @@ class Import::Gtfs < Import::Base
   end
 
   def import_agencies
+    count = 0
     Chouette::Company.transaction do
       source.agencies.each do |agency|
         company = line_referential.companies.find_or_initialize_by(registration_number: agency.id)
         company.attributes = { name: agency.name }
 
         save_model company
+        count += 1
       end
     end
+    create_message criticity: "info", message_key: "gtfs.agencies.imported", message_attributes: {count: count}
   end
 
   def import_stops
+    count = 0
     Chouette::StopArea.transaction do
       source.stops.each do |stop|
         stop_area = stop_area_referential.stop_areas.find_or_initialize_by(registration_number: stop.id)
@@ -155,11 +178,14 @@ class Import::Gtfs < Import::Base
         # TODO correct default timezone
 
         save_model stop_area
+        count += 1
       end
+      create_message criticity: "info", message_key: "gtfs.stops.imported", message_attributes: {count: count}
     end
   end
 
   def import_routes
+    count = 0
     Chouette::Line.transaction do
       source.routes.each do |route|
         line = line_referential.lines.find_or_initialize_by(registration_number: route.id)
@@ -178,7 +204,9 @@ class Import::Gtfs < Import::Base
         line.url = route.url
 
         save_model line
+        count += 1
       end
+      create_message criticity: "info", message_key: "gtfs.routes.imported", message_attributes: {count: count}
     end
   end
 
@@ -187,6 +215,7 @@ class Import::Gtfs < Import::Base
   end
 
   def import_trips
+    count = 0
     source.trips.each_slice(100) do |slice|
       slice.each do |trip|
         Chouette::Route.transaction do
@@ -205,18 +234,20 @@ class Import::Gtfs < Import::Base
           vehicle_journey = journey_pattern.vehicle_journeys.build route: route
           vehicle_journey.published_journey_name = trip.headsign.presence || trip.id
           save_model vehicle_journey
+          count += 1
 
           time_table = referential.time_tables.find_by(id: time_tables_by_service_id[trip.service_id]) if time_tables_by_service_id[trip.service_id]
           if time_table
             vehicle_journey.time_tables << time_table
           else
-            messages.create! criticity: "warning", message_key: "gtfs.trips.unkown_service_id", message_attributes: {service_id: trip.service_id}
+            create_message criticity: "warning", message_key: "gtfs.trips.unkown_service_id", message_attributes: {service_id: trip.service_id}
           end
 
           vehicle_journey_by_trip_id[trip.id] = vehicle_journey.id
         end
       end
     end
+    create_message criticity: "info", message_key: "gtfs.trips.imported", message_attributes: {count: count}
   end
 
   def import_stop_times
@@ -262,6 +293,7 @@ class Import::Gtfs < Import::Base
   end
 
   def import_calendars
+    count = 0
     source.calendars.each_slice(500) do |slice|
       Chouette::TimeTable.transaction do
         slice.each do |calendar|
@@ -272,11 +304,13 @@ class Import::Gtfs < Import::Base
           time_table.periods.build period_start: calendar.start_date, period_end: calendar.end_date
 
           save_model time_table
+          count += 1
 
           time_tables_by_service_id[calendar.service_id] = time_table.id
         end
       end
     end
+    create_message criticity: "info", message_key: "gtfs.calendars.imported", message_attributes: {count: count}
   end
 
   def import_calendar_dates
diff --git a/app/models/import/netex.rb b/app/models/import/netex.rb
index 49554ee90..e27f1a34c 100644
--- a/app/models/import/netex.rb
+++ b/app/models/import/netex.rb
@@ -4,6 +4,11 @@ class Import::Netex < Import::Base
 
   after_commit :call_iev_callback, on: :create
 
+  after_commit do
+    main_resource.update_status_from_importer self.status
+    true
+  end
+
   before_save do
     self.status = 'aborted' unless referential
     self.referential&.failed! if self.status == 'aborted' || self.status == 'failed'
@@ -11,7 +16,22 @@ class Import::Netex < Import::Base
 
   validates_presence_of :parent
 
+  def main_resource
+    @resource ||= parent.resources.find_or_create_by(name: self.name, resource_type: "referential", reference: self.name)
+  end
+
+  def notify_parent
+    super
+    main_resource.update_status_from_importer self.status
+  end
+
+  def create_message args
+    main_resource.messages.create args
+  end
+
   def create_with_referential!
+    save unless persisted?
+
     self.referential =
       Referential.new(
         name: self.name,
@@ -20,15 +40,35 @@ class Import::Netex < Import::Base
         metadatas: [referential_metadata]
       )
     self.referential.save
-    if self.referential.invalid?
+
+    if self.referential.valid?
+      main_resource.update referential: referential
+      call_iev_callback
+      save!
+    else
       Rails.logger.info "Can't create referential for import #{self.id}: #{referential.inspect} #{referential.metadatas.inspect} #{referential.errors.messages}"
-      if referential.metadatas.all?{|m| m.line_ids.present? && m.line_ids.empty?}
-        parent.messages.create criticity: :error, message_key: "referential_creation_missing_lines", message_attributes: {referential_name: referential.name}
+      aborted!
+      if referential.metadatas.all?{|m| m.line_ids.empty? && m.line_ids.empty?}
+        create_message criticity: :error, message_key: "referential_creation_missing_lines", message_attributes: {referential_name: referential.name}
+      elsif (overlapped_referential_ids = referential.overlapped_referential_ids).any?
+        overlapped = Referential.find overlapped_referential_ids.last
+        create_message(
+          criticity: :error,
+          message_key: "referential_creation_overlapping_existing_referential",
+          message_attributes: {
+            referential_name: referential.name,
+            overlapped_name: overlapped.name,
+            overlapped_url:  Rails.application.routes.url_helpers.referential_path(overlapped)
+          }
+        )
       else
-        parent.messages.create criticity: :error, message_key: "referential_creation", message_attributes: {referential_name: referential.name}
+        create_message(
+          criticity: :error,
+          message_key: "referential_creation",
+          message_attributes: {referential_name: referential.name},
+          resource_attributes: referential.errors.messages
+        )
       end
-    else
-      save!
     end
   end
 
@@ -55,6 +95,7 @@ class Import::Netex < Import::Base
         metadata.periodes = frame.periods
 
         line_objectids = frame.line_refs.map { |ref| "STIF:CODIFLIGNE:Line:#{ref}" }
+        create_message criticity: :info, message_key: "referential_creation_lines_found", message_attributes: {line_objectids: line_objectids.to_sentence}
         metadata.line_ids = workbench.lines.where(objectid: line_objectids).pluck(:id)
       end
     end
diff --git a/app/models/import/resource.rb b/app/models/import/resource.rb
index 1951daacd..e6de935ae 100644
--- a/app/models/import/resource.rb
+++ b/app/models/import/resource.rb
@@ -4,5 +4,17 @@ class Import::Resource < ApplicationModel
   include IevInterfaces::Resource
 
   belongs_to :import, class_name: Import::Base
+  belongs_to :referential
   has_many :messages, class_name: "Import::Message", foreign_key: :resource_id
+
+  def root_import
+    import = self.import
+    import = import.parent while import.parent
+    import
+  end
+
+  def netex_import
+    return unless self.resource_type == "referential"
+    import.children.where(name: self.reference).last
+  end
 end
diff --git a/app/models/import/workbench.rb b/app/models/import/workbench.rb
index 124b9b0d8..95d23fe5b 100644
--- a/app/models/import/workbench.rb
+++ b/app/models/import/workbench.rb
@@ -9,14 +9,18 @@ class Import::Workbench < Import::Base
     end
   end
 
+  # def main_resource
+  #   @resource ||= resources.find_or_create_by(name: self.name, resource_type: "workbench_import")
+  # end
+
   def import_gtfs
     update_column :status, 'running'
     update_column :started_at, Time.now
 
-    Import::Gtfs.create! parent_id: self.id, workbench: workbench, file: File.new(file.path), name: "Import GTFS", creator: "Web service"
+    Import::Gtfs.create! parent_type: self.class.name, parent_id: self.id, workbench: workbench, file: File.new(file.path), name: "Import GTFS", creator: "Web service"
 
-    update_column :status, 'successful'
-    update_column :ended_at, Time.now
+    # update_column :status, 'successful'
+    # update_column :ended_at, Time.now
   rescue Exception => e
     Rails.logger.error "Error while processing GTFS file: #{e}"
 
diff --git a/app/models/referential.rb b/app/models/referential.rb
index 792353a73..76b3bd84d 100644
--- a/app/models/referential.rb
+++ b/app/models/referential.rb
@@ -26,6 +26,7 @@ class Referential < ApplicationModel
 
   has_one :user
   has_many :api_keys, class_name: 'Api::V1::ApiKey', dependent: :destroy
+  has_many :import_resources, class_name: 'Import::Resource', dependent: :destroy
 
   belongs_to :organisation
   validates_presence_of :organisation
diff --git a/app/views/import_resources/show.html.slim b/app/views/import_resources/show.html.slim
new file mode 100644
index 000000000..7fd8b4456
--- /dev/null
+++ b/app/views/import_resources/show.html.slim
@@ -0,0 +1,52 @@
+- breadcrumb :import_resource, @import_resource
+
+.page_content.import_messages
+  .container-fluid
+    .row
+      .col-lg-12
+        - metadata = { 'Bilan d\'import' => link_to(@import_resource.root_import.name, workbench_import_path(@import_resource.root_import.workbench, @import_resource.root_import) ),
+            'Jeu de données associé' => ( @import_resource.referential.present? ? link_to(@import_resource.referential.name, referential_path(@import_resource.referential)) : '-' ) }
+        - metadata = metadata.update({t('.status') => import_status(@import_resource.status, verbose: true) })
+        = definition_list t('metadatas'), metadata
+
+
+      .col-lg-12
+        .error_messages
+          = render 'shared/iev_interfaces/messages', messages: @import_resource.messages
+
+
+      // XXX
+      //- if @import_resource.children.present?
+      - if @import_resource&.netex_import&.resources.present?
+        .col-lg-12
+          h2 = t('.table_title')
+        .col-lg-12
+          = t('.table_explanation')
+        .col-lg-12
+          = table_builder_2 @import_resource.netex_import.resources.where(resource_type: :file),
+            [ \
+              TableBuilderHelper::Column.new( \
+                key: :name, \
+                attribute: 'name', \
+                sortable: false, \
+              ), \
+              TableBuilderHelper::Column.new( \
+                key: :status, \
+                attribute: Proc.new { |n| import_resource_status(n.status) }, \
+                sortable: false, \
+              ), \
+              TableBuilderHelper::Column.new( \
+                name: 'Résultat des tests' , \
+                attribute: Proc.new { |n| I18n.t('import_resources.index.metrics', n.metrics.deep_symbolize_keys) }, \
+                sortable: false, \
+              ), \
+              TableBuilderHelper::Column.new( \
+                name: 'Téléchargement' , \
+                attribute: Proc.new { |n| '<i class="fa fa-download" aria-hidden="true"></i>'.html_safe }, \
+                sortable: false, \
+                link_to: lambda do |import_resource| \
+                  workbench_import_import_resource_import_messages_path(import_resource.import.workbench, import_resource.import, import_resource, format: 'csv' ) \
+                end \
+              ), \
+            ],
+            cls: 'table has-search'
diff --git a/app/views/imports/show.html.slim b/app/views/imports/show.html.slim
index 9d0a6423d..99458c7fe 100644
--- a/app/views/imports/show.html.slim
+++ b/app/views/imports/show.html.slim
@@ -6,55 +6,52 @@
   .container-fluid
     .row
       .col-lg-6.col-md-6.col-sm-12.col-xs-12
-        = definition_list t('metadatas'), { t('.data_recovery') => '-', t('.filename') => @import.try(:file_identifier)}
+        - metadata = { t('.data_recovery') => '-', t('.filename') => @import.try(:file_identifier)}
+        - metadata = metadata.update({t('.status') => import_status(@import.status, verbose: true) })
+        = definition_list t('metadatas'), metadata
 
-    .row
-      .col-lg-12
-        .error_messages
-          = render 'shared/iev_interfaces/messages', messages: @import.messages
+    .col-lg-12
+      .error_messages
+        = render 'shared/iev_interfaces/messages', messages: @import.messages
 
-    - if @import.children.any?
-      .row
-        .col-lg-12
-          = table_builder_2 @import.children,
-            [ \
-              TableBuilderHelper::Column.new( \
-                name: t('.referential_name'), \
-                attribute: 'name', \
-                sortable: false, \
-                link_to: lambda do |import| \
-                  referential_path(import.referential) if import.referential.present? \
-                end \
-              ), \
-              TableBuilderHelper::Column.new( \
-                key: :status, \
-                attribute: Proc.new { |n| import_status(n.status) }, \
-                sortable: false, \
-                link_to: lambda do |import| \
-                  workbench_import_import_resources_path(import.workbench_id, import) \
-                end \
-              ), \
-              TableBuilderHelper::Column.new( \
-                name: t('.stif_control'), \
-                attribute: '', \
-                sortable: false, \
-              ), \
-              TableBuilderHelper::Column.new( \
-                name: t('.organisation_control'), \
-                attribute: '', \
-                sortable: false, \
-              ) \
-            ],
-            cls: 'table',
-            overhead: [ \
-              {}, \
-              { \
-                title: I18n.t('imports.show.results', count: @import.children_succeedeed, total: @import.children.count), \
-                width: 1, \
-                cls: "#{@import.import_status_css_class} full-border" \
-              }, { \
-                title: I18n.t('imports.show.summary').html_safe, \
-                width: 2, \
-                cls: 'overheaded-default colspan="2"' \
-              } \
-            ]
+    - if @import.resources.any?
+      .col-lg-12
+        = table_builder_2 @import.resources,
+          [ \
+            TableBuilderHelper::Column.new( \
+              name: t('.referential_name'), \
+              attribute: 'name', \
+              sortable: false, \
+              link_to: lambda do |item| \
+                referential_path(item.referential) if item.referential.present? \
+              end \
+            ), \
+            TableBuilderHelper::Column.new( \
+              key: :status, \
+              attribute: Proc.new { |n| import_status(n.status) }, \
+              sortable: false, \
+              link_to: lambda do |item| \
+                workbench_import_import_resource_path(@import.workbench_id, @import, item) \
+              end \
+            ), \
+            TableBuilderHelper::Column.new( \
+              name: t('.stif_control'), \
+              attribute: '', \
+              sortable: false, \
+            ), \
+            TableBuilderHelper::Column.new( \
+              name: t('.organisation_control'), \
+              attribute: '', \
+              sortable: false, \
+            ) \
+          ],
+          cls: 'table',
+          overhead: [ \
+            {}, \
+            {}, \
+            { \
+              title: I18n.t('imports.show.summary').html_safe, \
+              width: 2, \
+              cls: 'overheaded-default colspan="2"' \
+            } \
+          ]
diff --git a/app/workers/workbench_import_worker/object_state_updater.rb b/app/workers/workbench_import_worker/object_state_updater.rb
index 1edc6b9a1..39d5f20b1 100644
--- a/app/workers/workbench_import_worker/object_state_updater.rb
+++ b/app/workers/workbench_import_worker/object_state_updater.rb
@@ -2,6 +2,11 @@
 class WorkbenchImportWorker
   module ObjectStateUpdater
 
+    def resource entry
+      @_resources ||= {}
+      @_resources[entry.name] ||= workbench_import.resources.find_or_create_by(name: entry.name, resource_type: "referential")
+    end
+
     def update_object_state entry, count
       workbench_import.update( total_steps: count )
       update_spurious entry
@@ -14,44 +19,48 @@ class WorkbenchImportWorker
 
     def update_foreign_lines entry
       return if entry.foreign_lines.empty?
-      workbench_import.messages.create(
+      resource(entry).messages.create(
         criticity: :error,
         message_key: 'foreign_lines_in_referential',
         message_attributes: {
           'source_filename' => workbench_import.file.file.file,
           'foreign_lines'   => entry.foreign_lines.join(', ')
         })
+      resource(entry).update status: :ERROR
     end
 
     def update_spurious entry
       return if entry.spurious.empty?
-      workbench_import.messages.create(
+      resource(entry).messages.create(
         criticity: :error,
         message_key: 'inconsistent_zip_file',
         message_attributes: {
           'source_filename' => workbench_import.file.file.file,
           'spurious_dirs'   => entry.spurious.join(', ')
         })
+      resource(entry).update status: :ERROR
     end
 
     def update_missing_calendar entry
       return unless entry.missing_calendar
-      workbench_import.messages.create(
+      resource(entry).messages.create(
         criticity: :error,
         message_key: 'missing_calendar_in_zip_file',
         message_attributes: {
           'source_filename' => entry.name
         })
+      resource(entry).update status: :ERROR
     end
 
     def update_wrong_calendar entry
       return unless entry.wrong_calendar
-      workbench_import.messages.create(
+      resource(entry).messages.create(
         criticity: :error,
         message_key: 'wrong_calendar_in_zip_file',
         message_attributes: {
           'source_filename' => entry.name
         })
+      resource(entry).update status: :ERROR
     end
   end
 end