diff options
50 files changed, 415 insertions, 485 deletions
diff --git a/app/controllers/api/v1/imports_controller.rb b/app/controllers/api/v1/imports_controller.rb index 3d7f4ca79..dc2df0697 100644 --- a/app/controllers/api/v1/imports_controller.rb +++ b/app/controllers/api/v1/imports_controller.rb @@ -1,11 +1,11 @@ class Api::V1::ImportsController < Api::V1::IbooController - defaults :resource_class => WorkbenchImport + defaults :resource_class => Import::Workbench belongs_to :workbench def create args = workbench_import_params.merge(creator: 'Webservice') @import = parent.workbench_imports.create(args) - if @import.valid? + if @import.valid? create! else render json: { status: "error", messages: @import.errors.full_messages } diff --git a/app/controllers/api/v1/internals/netex_imports_controller.rb b/app/controllers/api/v1/internals/netex_imports_controller.rb index c8e33f7b8..c2b7b20cc 100644 --- a/app/controllers/api/v1/internals/netex_imports_controller.rb +++ b/app/controllers/api/v1/internals/netex_imports_controller.rb @@ -25,13 +25,13 @@ module Api private def find_netex_import - @netex_import = NetexImport.find(params[:id]) + @netex_import = Import::Netex.find(params[:id]) rescue ActiveRecord::RecordNotFound render json: { - status: "error", + status: "error", message: "Record not found" } - finish_action! + finish_action! end def find_workbench @@ -52,7 +52,7 @@ module Api attributes = attributes.merge referential_id: @new_referential.id - @netex_import = NetexImport.new attributes + @netex_import = Import::Netex.new attributes @netex_import.save! unless @netex_import.referential diff --git a/app/controllers/api/v1/netex_imports_controller.rb b/app/controllers/api/v1/netex_imports_controller.rb index d86c1fcd8..2654fa088 100644 --- a/app/controllers/api/v1/netex_imports_controller.rb +++ b/app/controllers/api/v1/netex_imports_controller.rb @@ -34,7 +34,7 @@ module Api attributes = attributes.merge referential_id: @new_referential.id - @netex_import = NetexImport.new attributes + @netex_import = Import::Netex.new attributes @netex_import.save! unless @netex_import.referential diff --git a/app/controllers/exports_controller.rb b/app/controllers/exports_controller.rb index ccc163e34..095720aa0 100644 --- a/app/controllers/exports_controller.rb +++ b/app/controllers/exports_controller.rb @@ -3,72 +3,72 @@ require 'open-uri' class ExportsController < ChouetteController include ReferentialSupport - defaults :resource_class => Export - - respond_to :html, :only => [:show, :index, :destroy, :exported_file] - respond_to :js, :only => [:index] - belongs_to :referential - - def index - begin - index! - rescue Ievkit::Error, Faraday::Error => error - logger.error("Iev failure : #{error.message}") - flash[:error] = t(error.locale_for_error) - redirect_to referential_path(@referential) - end - end - - def show - begin - show! - rescue Ievkit::Error, Faraday::Error => error - logger.error("Iev failure : #{error.message}") - flash[:error] = t(error.locale_for_error) - redirect_to referential_path(@referential) - end - end - - def destroy - begin - destroy! - rescue Ievkit::Error, Faraday::Error => error - logger.error("Iev failure : #{error.message}") - flash[:error] = t(error.locale_for_error) - redirect_to referential_path(@referential) - end - end - - def exported_file - # WARNING : files under 10kb in size get treated as StringIO by OpenUri - # http://stackoverflow.com/questions/10496874/why-does-openuri-treat-files-under-10kb-in-size-as-stringio - OpenURI::Buffer.send :remove_const, 'StringMax' if OpenURI::Buffer.const_defined?('StringMax') - OpenURI::Buffer.const_set 'StringMax', 0 - begin - send_file open(resource.file_path), { :type => "application/#{resource.filename_extension}", :disposition => "attachment", :filename => resource.filename } - rescue Ievkit::Error, Faraday::Error => error - logger.error("Iev failure : #{error.message}") - flash[:error] = t(error.locale_for_error) - redirect_to referential_path(@referential) - end - end - - protected - - def export_service - ExportService.new(@referential) - end - - def resource - @export ||= export_service.find( params[:id] ) - @line_items = @export.report.line_items - if @line_items.size > 500 - @line_items = @line_items.paginate(page: params[:page], per_page: 20) - end - @export - end - - def collection - @exports ||= export_service.all.sort_by{ |export| export.created_at }.reverse.paginate(:page => params[:page]) - end + # defaults :resource_class => Export + # + # respond_to :html, :only => [:show, :index, :destroy, :exported_file] + # respond_to :js, :only => [:index] + # belongs_to :referential + # + # def index + # begin + # index! + # rescue Ievkit::Error, Faraday::Error => error + # logger.error("Iev failure : #{error.message}") + # flash[:error] = t(error.locale_for_error) + # redirect_to referential_path(@referential) + # end + # end + # + # def show + # begin + # show! + # rescue Ievkit::Error, Faraday::Error => error + # logger.error("Iev failure : #{error.message}") + # flash[:error] = t(error.locale_for_error) + # redirect_to referential_path(@referential) + # end + # end + # + # def destroy + # begin + # destroy! + # rescue Ievkit::Error, Faraday::Error => error + # logger.error("Iev failure : #{error.message}") + # flash[:error] = t(error.locale_for_error) + # redirect_to referential_path(@referential) + # end + # end + # + # def exported_file + # # WARNING : files under 10kb in size get treated as StringIO by OpenUri + # # http://stackoverflow.com/questions/10496874/why-does-openuri-treat-files-under-10kb-in-size-as-stringio + # OpenURI::Buffer.send :remove_const, 'StringMax' if OpenURI::Buffer.const_defined?('StringMax') + # OpenURI::Buffer.const_set 'StringMax', 0 + # begin + # send_file open(resource.file_path), { :type => "application/#{resource.filename_extension}", :disposition => "attachment", :filename => resource.filename } + # rescue Ievkit::Error, Faraday::Error => error + # logger.error("Iev failure : #{error.message}") + # flash[:error] = t(error.locale_for_error) + # redirect_to referential_path(@referential) + # end + # end + # + # protected + # + # def export_service + # ExportService.new(@referential) + # end + # + # def resource + # @export ||= export_service.find( params[:id] ) + # @line_items = @export.report.line_items + # if @line_items.size > 500 + # @line_items = @line_items.paginate(page: params[:page], per_page: 20) + # end + # @export + # end + # + # def collection + # @exports ||= export_service.all.sort_by{ |export| export.created_at }.reverse.paginate(:page => params[:page]) + # end end diff --git a/app/controllers/import_messages_controller.rb b/app/controllers/import_messages_controller.rb index 4f8fe7a25..5e39445fb 100644 --- a/app/controllers/import_messages_controller.rb +++ b/app/controllers/import_messages_controller.rb @@ -1,8 +1,8 @@ class ImportMessagesController < ChouetteController - defaults resource_class: ImportMessage, collection_name: 'import_messages', instance_name: 'import_message' + defaults resource_class: Import::Message, collection_name: 'import_messages', instance_name: 'import_message' respond_to :csv - belongs_to :import, :parent_class => Import do - belongs_to :import_resource, :parent_class => ImportResource + belongs_to :import, :parent_class => Import::Base do + belongs_to :import_resource, :parent_class => Import::Resource end @@ -20,7 +20,7 @@ class ImportMessagesController < ChouetteController end def parent - @import_resource ||= ImportResource.find(params[:import_resource_id]) + @import_resource ||= Import::Resource.find(params[:import_resource_id]) end end diff --git a/app/controllers/import_resources_controller.rb b/app/controllers/import_resources_controller.rb index ea78394a1..6d1977e0c 100644 --- a/app/controllers/import_resources_controller.rb +++ b/app/controllers/import_resources_controller.rb @@ -1,5 +1,5 @@ class ImportResourcesController < ChouetteController - defaults resource_class: ImportResource, collection_name: 'import_resources', instance_name: 'import_resource' + defaults resource_class: Import::Resource, collection_name: 'import_resources', instance_name: 'import_resource' respond_to :html belongs_to :import diff --git a/app/controllers/import_tasks_controller.rb b/app/controllers/import_tasks_controller.rb deleted file mode 100644 index 1a349087d..000000000 --- a/app/controllers/import_tasks_controller.rb +++ /dev/null @@ -1,69 +0,0 @@ -class ImportTasksController < ChouetteController - include ReferentialSupport - defaults :resource_class => ImportTask - - respond_to :html, :only => [:new, :create] - respond_to :js, :only => [:new, :create] - belongs_to :referential - - def new - @available_imports = available_imports - begin - new! - rescue Ievkit::Error, Faraday::Error => error - logger.error("Iev failure : #{error.message}") - flash[:error] = t('iev.exception.default') - redirect_to referential_path(@referential) - end - end - - def create - @available_imports = available_imports - begin - create! do |success, failure| - success.html { redirect_to referential_imports_path(@referential) } - end - rescue Ievkit::Error, Faraday::Error => error - logger.error("Iev failure : #{error.message}") - flash[:error] = t('iev.exception.default') - redirect_to referential_path(@referential) - end - end - - protected - - def available_imports - import_task_parameters = params[:import_task] - - if import_task_parameters.present? - @available_imports = [ - import_task_parameters[:data_format] == "neptune" ? build_resource : NeptuneImport.new(:referential_id => @referential.id ), - import_task_parameters[:data_format] == "netex" ? build_resource : NetexImport.new(:referential_id => @referential.id ), - import_task_parameters[:data_format] == "gtfs" ? build_resource : GtfsImport.new(:referential_id => @referential.id ) - ] - else - @available_imports = [ - NeptuneImport.new(:referential_id => @referential.id ), - NetexImport.new(:referential_id => @referential.id ), - GtfsImport.new(:referential_id => @referential.id ) - ] - end - end - - def build_resource - @import_task ||= if params[:import_task].present? - import_task_parameters = params[:import_task] - case import_task_parameters[:data_format] - when "neptune" - NeptuneImport.new(import_task_parameters) - when "netex" - @import_task = NetexImport.new(import_task_parameters) - when "gtfs" - @import_task = GtfsImport.new(import_task_parameters) - end - else - @import_task = NeptuneImport.new - end - end - -end diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index 7a999d657..3ebbeb665 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -3,7 +3,7 @@ class ImportsController < ChouetteController include RansackDateFilter before_action only: [:index] { set_date_time_params("started_at", DateTime) } skip_before_action :authenticate_user!, only: [:download] - defaults resource_class: Import, collection_name: 'imports', instance_name: 'import' + defaults resource_class: Import::Base, collection_name: 'imports', instance_name: 'import' before_action :ransack_status_params, only: [:index] respond_to :html belongs_to :workbench @@ -37,7 +37,7 @@ class ImportsController < ChouetteController protected def collection - scope = parent.imports.where(type: "WorkbenchImport") + scope = parent.imports.where(type: "Import::Workbench") scope = self.ransack_period_range(scope: scope, error_message: t('imports.filters.error_period_filter'), query: :where_started_at_in) @@ -54,14 +54,14 @@ class ImportsController < ChouetteController def ransack_status_params if params[:q] - return params[:q].delete(:status_eq_any) if params[:q][:status_eq_any].empty? || ( (Import.status.values & params[:q][:status_eq_any]).length >= 4 ) + return params[:q].delete(:status_eq_any) if params[:q][:status_eq_any].empty? || ( (Import::Base.status.values & params[:q][:status_eq_any]).length >= 4 ) params[:q][:status_eq_any].push("new", "running") if params[:q][:status_eq_any].include?("pending") params[:q][:status_eq_any].push("aborted", "canceled") if params[:q][:status_eq_any].include?("failed") end end def build_resource - @import ||= WorkbenchImport.new(*resource_params) do |import| + @import ||= Import::Workbench.new(*resource_params) do |import| import.workbench = parent import.creator = current_user.name end diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb index e38a92982..152da4fa2 100644 --- a/app/controllers/statuses_controller.rb +++ b/app/controllers/statuses_controller.rb @@ -5,7 +5,7 @@ class StatusesController < ChouetteController status = { referentials_blocked: Referential.blocked.count, - imports_blocked: Import.blocked.count, + imports_blocked: Import::Base.blocked.count, compliance_check_sets_blocked: ComplianceCheckSet.blocked.count } status[:status] = global_status status diff --git a/app/decorators/import_decorator.rb b/app/decorators/import/base_decorator.rb index 1964365ae..8c10479c6 100644 --- a/app/decorators/import_decorator.rb +++ b/app/decorators/import/base_decorator.rb @@ -1,5 +1,5 @@ -class ImportDecorator < AF83::Decorator - decorates Import +class Import::BaseDecorator < AF83::Decorator + decorates Import::Base set_scope { context[:workbench] } diff --git a/app/models/concerns/iev_interfaces/message.rb b/app/models/concerns/iev_interfaces/message.rb new file mode 100644 index 000000000..ad41e98b7 --- /dev/null +++ b/app/models/concerns/iev_interfaces/message.rb @@ -0,0 +1,9 @@ +module IevInterfaces::Message + extend ActiveSupport::Concern + + included do + extend Enumerize + enumerize :criticity, in: %i(info warning error) + validates :criticity, presence: true + end +end diff --git a/app/models/concerns/iev_interfaces/resource.rb b/app/models/concerns/iev_interfaces/resource.rb new file mode 100644 index 000000000..7f8c3eefd --- /dev/null +++ b/app/models/concerns/iev_interfaces/resource.rb @@ -0,0 +1,9 @@ +module IevInterfaces::Resource + extend ActiveSupport::Concern + + included do + extend Enumerize + enumerize :status, in: %i(OK ERROR WARNING IGNORED), scope: true + validates_presence_of :name, :resource_type, :reference + end +end diff --git a/app/models/concerns/iev_interfaces/task.rb b/app/models/concerns/iev_interfaces/task.rb new file mode 100644 index 000000000..2eb0c532f --- /dev/null +++ b/app/models/concerns/iev_interfaces/task.rb @@ -0,0 +1,51 @@ +module IevInterfaces::Task + extend ActiveSupport::Concern + + included do + belongs_to :parent, polymorphic: true + belongs_to :workbench, class_name: "::Workbench" + belongs_to :referential + + extend Enumerize + enumerize :status, in: %w(new pending successful warning failed running aborted canceled), scope: true, default: :new + + validates :name, presence: true + validates_presence_of :workbench, :creator + + has_many :messages, class_name: messages_class_name, dependent: :destroy, foreign_key: :import_id + has_many :resources, class_name: resources_class_name, dependent: :destroy, foreign_key: :import_id + + scope :where_started_at_in, ->(period_range) do + where('started_at BETWEEN :begin AND :end', begin: period_range.begin, end: period_range.end) + end + + scope :blocked, -> { where('created_at < ? AND status = ?', 4.hours.ago, 'running') } + end + + module ClassMethods + def launched_statuses + %w(new pending) + end + + def failed_statuses + %w(failed aborted canceled) + end + + def finished_statuses + %w(successful failed warning aborted canceled) + end + + def abort_old + where( + 'created_at < ? AND status NOT IN (?)', + 4.hours.ago, + finished_statuses + ).update_all(status: 'aborted') + end + end + + def notify_parent + parent.child_change + update(notified_parent_at: DateTime.now) + end +end diff --git a/app/models/import.rb b/app/models/import.rb deleted file mode 100644 index adad39b59..000000000 --- a/app/models/import.rb +++ /dev/null @@ -1,103 +0,0 @@ -class Import < ActiveRecord::Base - mount_uploader :file, ImportUploader - belongs_to :workbench - belongs_to :referential - - belongs_to :parent, polymorphic: true - - has_many :messages, class_name: "ImportMessage", dependent: :destroy - has_many :resources, class_name: "ImportResource", dependent: :destroy - has_many :children, foreign_key: :parent_id, class_name: "Import", dependent: :destroy - - scope :where_started_at_in, ->(period_range) do - where('started_at BETWEEN :begin AND :end', begin: period_range.begin, end: period_range.end) - end - - scope :blocked, -> { where('created_at < ? AND status = ?', 4.hours.ago, 'running') } - - extend Enumerize - enumerize :status, in: %w(new pending successful warning failed running aborted canceled), scope: true, default: :new - - validates :name, presence: true - validates :file, presence: true - validates_presence_of :workbench, :creator - - before_create :initialize_fields - - def self.model_name - ActiveModel::Name.new Import, Import, "Import" - end - - def children_succeedeed - children.with_status(:successful, :warning).count - end - - def self.launched_statuses - %w(new pending) - end - - def self.failed_statuses - %w(failed aborted canceled) - end - - def self.finished_statuses - %w(successful failed warning aborted canceled) - end - - def self.abort_old - where( - 'created_at < ? AND status NOT IN (?)', - 4.hours.ago, - finished_statuses - ).update_all(status: 'aborted') - end - - def notify_parent - parent.child_change - update(notified_parent_at: DateTime.now) - end - - def child_change - return if self.class.finished_statuses.include?(status) - - update_status - update_referentials - end - - def update_status - status = - if children.where(status: self.class.failed_statuses).count > 0 - 'failed' - elsif children.where(status: "warning").count > 0 - 'warning' - elsif children.where(status: "successful").count == children.count - 'successful' - end - - attributes = { - current_step: children.count, - status: status - } - - if self.class.finished_statuses.include?(status) - attributes[:ended_at] = Time.now - end - - update attributes - end - - def update_referentials - return unless self.class.finished_statuses.include?(status) - - children.each do |import| - import.referential.update(ready: true) if import.referential - end - end - - private - - def initialize_fields - self.token_download = SecureRandom.urlsafe_base64 - end - -end diff --git a/app/models/import/base.rb b/app/models/import/base.rb new file mode 100644 index 000000000..0e1eae2c0 --- /dev/null +++ b/app/models/import/base.rb @@ -0,0 +1,73 @@ +class Import::Base < ActiveRecord::Base + self.table_name = "imports" + + def self.messages_class_name + "Import::Message" + end + + def self.resources_class_name + "Import::Resource" + end + + include IevInterfaces::Task + + mount_uploader :file, ImportUploader + + has_many :children, foreign_key: :parent_id, class_name: "Import::Base", dependent: :destroy + + validates :file, presence: true + + before_create :initialize_fields + + def self.model_name + ActiveModel::Name.new Import::Base, Import::Base, "Import::Base" + end + + def children_succeedeed + children.with_status(:successful, :warning).count + end + + def child_change + return if self.class.finished_statuses.include?(status) + + update_status + update_referentials + end + + def update_status + status = + if children.where(status: self.class.failed_statuses).count > 0 + 'failed' + elsif children.where(status: "warning").count > 0 + 'warning' + elsif children.where(status: "successful").count == children.count + 'successful' + end + + attributes = { + current_step: children.count, + status: status + } + + if self.class.finished_statuses.include?(status) + attributes[:ended_at] = Time.now + end + + update attributes + end + + def update_referentials + return unless self.class.finished_statuses.include?(status) + + children.each do |import| + import.referential.update(ready: true) if import.referential + end + end + + private + + def initialize_fields + self.token_download = SecureRandom.urlsafe_base64 + end + +end diff --git a/app/models/gtfs_import.rb b/app/models/import/gtfs.rb index d09ca4cb3..03cf49e60 100644 --- a/app/models/gtfs_import.rb +++ b/app/models/import/gtfs.rb @@ -1,5 +1,5 @@ require 'net/http' -class GtfsImport < Import +class Import::Gtfs < Import::Base before_destroy :destroy_non_ready_referential after_commit :launch_java_import, on: :create diff --git a/app/models/import/message.rb b/app/models/import/message.rb new file mode 100644 index 000000000..c1900a718 --- /dev/null +++ b/app/models/import/message.rb @@ -0,0 +1,8 @@ +class Import::Message < ActiveRecord::Base + self.table_name = :import_messages + + include IevInterfaces::Message + + belongs_to :import, class_name: Import::Base + belongs_to :resource, class_name: Import::Resource +end diff --git a/app/models/netex_import.rb b/app/models/import/netex.rb index b21af3408..a4bf0920d 100644 --- a/app/models/netex_import.rb +++ b/app/models/import/netex.rb @@ -1,5 +1,5 @@ require 'net/http' -class NetexImport < Import +class Import::Netex < Import::Base before_destroy :destroy_non_ready_referential after_commit :launch_java_import, on: :create diff --git a/app/models/import/resource.rb b/app/models/import/resource.rb new file mode 100644 index 000000000..73be04d0e --- /dev/null +++ b/app/models/import/resource.rb @@ -0,0 +1,8 @@ +class Import::Resource < ActiveRecord::Base + self.table_name = :import_resources + + include IevInterfaces::Resource + + belongs_to :import, class_name: Import::Base + has_many :messages, class_name: "ImportMessage", foreign_key: :resource_id +end diff --git a/app/models/workbench_import.rb b/app/models/import/workbench.rb index 27f53a44f..f6e15cb89 100644 --- a/app/models/workbench_import.rb +++ b/app/models/import/workbench.rb @@ -1,4 +1,4 @@ -class WorkbenchImport < Import +class Import::Workbench < Import::Base after_commit :launch_worker, :on => :create def launch_worker diff --git a/app/models/import_message.rb b/app/models/import_message.rb deleted file mode 100644 index de70c35d1..000000000 --- a/app/models/import_message.rb +++ /dev/null @@ -1,8 +0,0 @@ -class ImportMessage < ActiveRecord::Base - extend Enumerize - belongs_to :import - belongs_to :resource, class_name: ImportResource - enumerize :criticity, in: %i(info warning error) - - validates :criticity, presence: true -end diff --git a/app/models/import_resource.rb b/app/models/import_resource.rb deleted file mode 100644 index 55e752e74..000000000 --- a/app/models/import_resource.rb +++ /dev/null @@ -1,11 +0,0 @@ -class ImportResource < ActiveRecord::Base - belongs_to :import - - extend Enumerize - enumerize :status, in: %i(OK ERROR WARNING IGNORED), scope: true - - validates_presence_of :name, :resource_type, :reference - - has_many :messages, class_name: "ImportMessage", foreign_key: :resource_id - -end diff --git a/app/models/workbench.rb b/app/models/workbench.rb index b6f90c7dc..44415f102 100644 --- a/app/models/workbench.rb +++ b/app/models/workbench.rb @@ -13,8 +13,8 @@ class Workbench < ActiveRecord::Base has_many :companies, through: :line_referential has_many :group_of_lines, through: :line_referential has_many :stop_areas, through: :stop_area_referential - has_many :imports - has_many :workbench_imports + has_many :imports, class_name: Import::Base + has_many :workbench_imports, class_name: Import::Workbench has_many :compliance_check_sets has_many :compliance_control_sets has_many :merges diff --git a/app/policies/import_policy.rb b/app/policies/import/base_policy.rb index 512bfc8ac..62345e452 100644 --- a/app/policies/import_policy.rb +++ b/app/policies/import/base_policy.rb @@ -1,4 +1,4 @@ -class ImportPolicy < ApplicationPolicy +class Import::BasePolicy < ApplicationPolicy class Scope < Scope def resolve scope diff --git a/app/views/compliance_control_sets/_filters.html.slim b/app/views/compliance_control_sets/_filters.html.slim index 4348defac..5cf282559 100644 --- a/app/views/compliance_control_sets/_filters.html.slim +++ b/app/views/compliance_control_sets/_filters.html.slim @@ -11,7 +11,7 @@ = f.input :organisation_name_eq_any, collection: organisations_filters_values, as: :check_boxes, label: false, label_method: lambda {|w| ("<span>#{w.name}</span>").html_safe}, required: false, wrapper_html: {class: 'checkbox_list'} .form-group.togglable class=filter_item_class(params[:q], :updated_at) - = f.label Import.human_attribute_name(:updated_at), required: false, class: 'control-label' + = f.label Import::Base.human_attribute_name(:updated_at), required: false, class: 'control-label' .filter_menu = f.simple_fields_for :updated_at do |p| = p.input :start_date, as: :date, label: false, wrapper_html: {class: 'date smart_date filter_menu-item'}, default: @begin_range, include_blank: @begin_range ? false : true diff --git a/app/workers/workbench_import_worker.rb b/app/workers/workbench_import_worker.rb index 53cbb222a..fd2a888f0 100644 --- a/app/workers/workbench_import_worker.rb +++ b/app/workers/workbench_import_worker.rb @@ -12,7 +12,7 @@ class WorkbenchImportWorker def perform(import_id) @entries = 0 - @workbench_import ||= WorkbenchImport.find(import_id) + @workbench_import ||= Import::Workbench.find(import_id) workbench_import.update(status: 'running', started_at: Time.now) zip_service = ZipService.new(downloaded, allowed_lines) @@ -44,7 +44,6 @@ class WorkbenchImportWorker raise end - def upload_entry_group entry, element_count update_object_state entry, element_count.succ return unless entry.ok? @@ -80,7 +79,6 @@ class WorkbenchImportWorker File.unlink(eg_file.path) end - # Queries # ======= diff --git a/config/initializers/apartment.rb b/config/initializers/apartment.rb index f5fb8cd5e..53b27006a 100644 --- a/config/initializers/apartment.rb +++ b/config/initializers/apartment.rb @@ -18,72 +18,73 @@ Apartment.configure do |config| # config.excluded_models = %w{Tenant} # config.excluded_models = [ - 'Referential', - 'ReferentialMetadata', - 'ReferentialSuite', - 'Organisation', - 'User', 'Api::V1::ApiKey', - 'StopAreaReferential', - 'StopAreaReferentialMembership', - 'StopAreaReferentialSync', - 'StopAreaReferentialSyncMessage', - 'Chouette::StopArea', - 'LineReferential', - 'LineReferentialMembership', - 'LineReferentialSync', - 'LineReferentialSyncMessage', - 'Chouette::Line', - 'Chouette::GroupOfLine', + 'Calendar', 'Chouette::Company', + 'Chouette::GroupOfLine', + 'Chouette::Line', 'Chouette::Network', - 'ReferentialCloning', - 'Workbench', - 'Workgroup', + 'Chouette::StopArea', 'CleanUp', 'CleanUpResult', - 'Calendar', - 'Import', - 'NetexImport', - 'WorkbenchImport', - 'ImportMessage', - 'ImportResource', + 'ComplianceCheck', + 'ComplianceCheckBlock', + 'ComplianceCheckMessage', + 'ComplianceCheckResource', + 'ComplianceCheckSet', 'ComplianceControl', + 'ComplianceControlBlock', + 'ComplianceControlSet', + 'CustomField', 'GenericAttributeControl::MinMax', 'GenericAttributeControl::Pattern', 'GenericAttributeControl::Uniqueness', + 'Import::Base', + 'Import::Gtfs', + 'Import::Message', + 'Import::Netex', + 'Import::Resource', + 'Import::Workbench', 'JourneyPatternControl::Duplicates', 'JourneyPatternControl::VehicleJourney', 'LineControl::Route', + 'LineReferential', + 'LineReferentialMembership', + 'LineReferentialSync', + 'LineReferentialSyncMessage', + 'Merge', + 'Organisation', + 'Referential', + 'ReferentialCloning', + 'ReferentialMetadata', + 'ReferentialSuite', 'RouteControl::Duplicates', 'RouteControl::JourneyPattern', 'RouteControl::MinimumLength', 'RouteControl::OmnibusJourneyPattern', - 'RouteControl::OppositeRouteTerminus', 'RouteControl::OppositeRoute', + 'RouteControl::OppositeRouteTerminus', 'RouteControl::StopPointsInJourneyPattern', 'RouteControl::UnactivatedStopPoint', 'RouteControl::ZDLStopArea', 'RoutingConstraintZoneControl::MaximumLength', 'RoutingConstraintZoneControl::MinimumLength', 'RoutingConstraintZoneControl::UnactivatedStopPoint', + 'SimpleExporter', + 'SimpleImporter', + 'SimpleInterface', + 'StopAreaReferential', + 'StopAreaReferentialMembership', + 'StopAreaReferentialSync', + 'StopAreaReferentialSyncMessage', + 'User', 'VehicleJourneyControl::Delta', - 'VehicleJourneyControl::WaitingTime', 'VehicleJourneyControl::Speed', 'VehicleJourneyControl::TimeTable', 'VehicleJourneyControl::VehicleJourneyAtStops', - 'ComplianceControlSet', - 'ComplianceControlBlock', - 'ComplianceCheck', - 'ComplianceCheckSet', - 'ComplianceCheckBlock', - 'ComplianceCheckResource', - 'ComplianceCheckMessage', - 'Merge', - 'CustomField', - 'SimpleInterface', - 'SimpleImporter', - 'SimpleExporter', + 'VehicleJourneyControl::WaitingTime', + 'Workbench', + 'Workgroup', ] # use postgres schemas? diff --git a/db/migrate/20171106111448_update_import_message_criticity_type_to_string.rb b/db/migrate/20171106111448_update_import_message_criticity_type_to_string.rb index c14450387..a08a10b9a 100644 --- a/db/migrate/20171106111448_update_import_message_criticity_type_to_string.rb +++ b/db/migrate/20171106111448_update_import_message_criticity_type_to_string.rb @@ -7,10 +7,10 @@ class UpdateImportMessageCriticityTypeToString < ActiveRecord::Migration when 0 then "info" when 1 then "warning" when 2 then "error" - else + else "info" end end - ImportMessage.all.each { |im| im.update_attribute(:criticity, change_criticity_value(im.criticity)) } + Import::Message.all.each { |im| im.update_attribute(:criticity, change_criticity_value(im.criticity)) } end end diff --git a/db/migrate/20180306152953_update_imports_names.rb b/db/migrate/20180306152953_update_imports_names.rb new file mode 100644 index 000000000..086d5ce5a --- /dev/null +++ b/db/migrate/20180306152953_update_imports_names.rb @@ -0,0 +1,8 @@ +class UpdateImportsNames < ActiveRecord::Migration + def change + Import::Base.all.pluck(:type).uniq.each do |type| + next if type =~ /^Imports/ + Import::Base.where(type: type).update_all type: "Import::#{type.gsub 'Import', ''}" + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 276d7f79a..53dbaceb2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,14 +11,13 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180306135204) do +ActiveRecord::Schema.define(version: 20180306152953) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" - enable_extension "hstore" enable_extension "postgis" + enable_extension "hstore" enable_extension "unaccent" - enable_extension "objectid" create_table "access_links", id: :bigserial, force: :cascade do |t| t.integer "access_point_id", limit: 8 @@ -91,9 +90,9 @@ ActiveRecord::Schema.define(version: 20180306135204) do t.integer "organisation_id", limit: 8 t.datetime "created_at" t.datetime "updated_at" + t.integer "workgroup_id", limit: 8 t.integer "int_day_types" t.date "excluded_dates", array: true - t.integer "workgroup_id", limit: 8 end add_index "calendars", ["organisation_id"], name: "index_calendars_on_organisation_id", using: :btree @@ -120,7 +119,6 @@ ActiveRecord::Schema.define(version: 20180306135204) do t.datetime "updated_at" t.date "end_date" t.string "date_type" - t.string "mode" end add_index "clean_ups", ["referential_id"], name: "index_clean_ups_on_referential_id", using: :btree @@ -755,7 +753,6 @@ ActiveRecord::Schema.define(version: 20180306135204) do t.datetime "created_at" t.datetime "updated_at" t.string "objectid_format" - t.string "registration_number_format" end create_table "stop_areas", id: :bigserial, force: :cascade do |t| diff --git a/spec/controllers/api/v1/imports_controller_spec.rb b/spec/controllers/api/v1/imports_controller_spec.rb index 8077dd052..f7022115a 100644 --- a/spec/controllers/api/v1/imports_controller_spec.rb +++ b/spec/controllers/api/v1/imports_controller_spec.rb @@ -30,7 +30,7 @@ RSpec.describe Api::V1::ImportsController, type: :controller do it 'should be successful' do expect { post :create, workbench_id: workbench.id, workbench_import: {name: "test", file: file, creator: 'test'}, format: :json - }.to change{WorkbenchImport.count}.by(1) + }.to change{Import::Workbench.count}.by(1) expect(response).to be_success end end diff --git a/spec/controllers/vehicle_journey_imports_controller_spec.rb b/spec/controllers/vehicle_journey_imports_controller_spec.rb deleted file mode 100644 index 633f90b70..000000000 --- a/spec/controllers/vehicle_journey_imports_controller_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'spec_helper' - -describe ImportTasksController, :type => :controller do - login_user -end diff --git a/spec/factories/import_messages.rb b/spec/factories/imports/import_messages.rb index 5d936679a..f5edf1685 100644 --- a/spec/factories/import_messages.rb +++ b/spec/factories/imports/import_messages.rb @@ -1,5 +1,5 @@ FactoryGirl.define do - factory :import_message do + factory :import_message, class: Import::Message do association :import association :resource, factory: :import_resource criticity :info diff --git a/spec/factories/import_resources.rb b/spec/factories/imports/import_resources.rb index 76afcc486..aaf7e3111 100644 --- a/spec/factories/import_resources.rb +++ b/spec/factories/imports/import_resources.rb @@ -1,5 +1,5 @@ FactoryGirl.define do - factory :import_resource do + factory :import_resource, class: Import::Resource do association :import status :WARNING sequence(:name) { |n| "Import resource #{n}" } diff --git a/spec/factories/imports.rb b/spec/factories/imports/imports.rb index e07447b60..cb7764cc6 100644 --- a/spec/factories/imports.rb +++ b/spec/factories/imports/imports.rb @@ -1,5 +1,5 @@ FactoryGirl.define do - factory :import do + factory :import, class: Import::Base do sequence(:name) { |n| "Import #{n}" } current_step_id "MyString" current_step_progress 1.5 @@ -16,7 +16,7 @@ FactoryGirl.define do end end - factory :bad_import do + factory :bad_import, class: Import::Base do sequence(:name) { |n| "Import #{n}" } current_step_id "MyString" current_step_progress 1.5 diff --git a/spec/factories/netex_imports.rb b/spec/factories/imports/netex_imports.rb index b59267a0a..7ee6839e8 100644 --- a/spec/factories/netex_imports.rb +++ b/spec/factories/imports/netex_imports.rb @@ -1,7 +1,7 @@ FactoryGirl.define do - factory :netex_import, class: NetexImport, parent: :import do + factory :netex_import, class: Import::Netex, parent: :import do file { File.open(Rails.root.join('spec', 'fixtures', 'OFFRE_TRANSDEV_2017030112251.zip')) } association :parent, factory: :workbench_import - + end end diff --git a/spec/factories/workbench_imports.rb b/spec/factories/imports/workbench_imports.rb index 466bfe688..5ed1ee4e5 100644 --- a/spec/factories/workbench_imports.rb +++ b/spec/factories/imports/workbench_imports.rb @@ -1,5 +1,5 @@ FactoryGirl.define do - factory :workbench_import, class: WorkbenchImport, parent: :import do + factory :workbench_import, class: Import::Workbench, parent: :import do file { File.open(Rails.root.join('spec', 'fixtures', 'OFFRE_TRANSDEV_2017030112251.zip')) } end end diff --git a/spec/models/export_log_message_spec.rb b/spec/models/export_log_message_spec.rb deleted file mode 100644 index 5ab32dec0..000000000 --- a/spec/models/export_log_message_spec.rb +++ /dev/null @@ -1,16 +0,0 @@ -require 'spec_helper' - -describe ExportLogMessage, :type => :model do - - # describe "#attributes" do - - # subject { create :export_log_message } - - # it "should read json stored in database" do - # subject.update_attribute :arguments, { "key" => "value"} - # expect(subject.raw_attributes).to eq({ "key" => "value"}.to_json) - # end - - # end - -end diff --git a/spec/models/gtfs_export_spec.rb b/spec/models/gtfs_export_spec.rb index ccc98e872..0ef3660f5 100644 --- a/spec/models/gtfs_export_spec.rb +++ b/spec/models/gtfs_export_spec.rb @@ -1,33 +1,33 @@ require 'spec_helper' -describe GtfsExport, :type => :model do - - describe "#time_zone" do - - context "when exported data are not StopAreas" do - - before do - subject.references_type = "network" - end - - it "should be mandatory" do - should validate_presence_of(:time_zone) - end - - end - - context "when export data are StopArea" do - - before do - subject.references_type = "stop_area" - end - - it "should be mandatory" do - should_not validate_presence_of(:time_zone) - end - - end - - end - -end +# describe GtfsExport, :type => :model do +# +# describe "#time_zone" do +# +# context "when exported data are not StopAreas" do +# +# before do +# subject.references_type = "network" +# end +# +# it "should be mandatory" do +# should validate_presence_of(:time_zone) +# end +# +# end +# +# context "when export data are StopArea" do +# +# before do +# subject.references_type = "stop_area" +# end +# +# it "should be mandatory" do +# should_not validate_presence_of(:time_zone) +# end +# +# end +# +# end +# +# end diff --git a/spec/models/gtfs_import_spec.rb b/spec/models/gtfs_import_spec.rb index 07cc1905d..5cb69332c 100644 --- a/spec/models/gtfs_import_spec.rb +++ b/spec/models/gtfs_import_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe GtfsImport, :type => :model do +describe Import::Gtfs, :type => :model do # describe "#object_id_prefix" do diff --git a/spec/models/import/netex_import_spec.rb b/spec/models/import/netex_import_spec.rb index 8ffeed1f4..9b9f50117 100644 --- a/spec/models/import/netex_import_spec.rb +++ b/spec/models/import/netex_import_spec.rb @@ -1,4 +1,4 @@ -RSpec.describe NetexImport, type: [:model, :with_commit] do +RSpec.describe Import::Netex, type: [:model, :with_commit] do let( :boiv_iev_uri ){ URI("#{Rails.configuration.iev_url}/boiv_iev/referentials/importer/new?id=#{subject.id}")} @@ -30,4 +30,42 @@ RSpec.describe NetexImport, type: [:model, :with_commit] do end end + describe "#destroy" do + it "must destroy its associated Referential if ready: false" do + workbench_import = create(:workbench_import) + referential_ready_false = create(:referential, ready: false) + referential_ready_true = create(:referential, ready: true) + create( + :netex_import, + parent: workbench_import, + referential: referential_ready_false + ) + create( + :netex_import, + parent: workbench_import, + referential: referential_ready_true + ) + + workbench_import.destroy + + expect( + Referential.where(id: referential_ready_false.id).exists? + ).to be false + expect( + Referential.where(id: referential_ready_true.id).exists? + ).to be true + end + + it "doesn't try to destroy nil referentials" do + workbench_import = create(:workbench_import) + create( + :netex_import, + parent: workbench_import, + referential: nil + ) + + expect { workbench_import.destroy }.not_to raise_error + end + end + end diff --git a/spec/models/import_message_spec.rb b/spec/models/import_message_spec.rb index 2d8aac2b7..48e03a2cc 100644 --- a/spec/models/import_message_spec.rb +++ b/spec/models/import_message_spec.rb @@ -1,6 +1,6 @@ require 'rails_helper' -RSpec.describe ImportMessage, :type => :model do +RSpec.describe Import::Message, :type => :model do it { should validate_presence_of(:criticity) } it { should belong_to(:import) } it { should belong_to(:resource) } diff --git a/spec/models/import_resource_spec.rb b/spec/models/import_resource_spec.rb index c88bb5dd2..7d2eab8f1 100644 --- a/spec/models/import_resource_spec.rb +++ b/spec/models/import_resource_spec.rb @@ -1,6 +1,6 @@ require 'rails_helper' -RSpec.describe ImportResource, :type => :model do +RSpec.describe Import::Resource, :type => :model do it { should belong_to(:import) } it { should enumerize(:status).in("OK", "ERROR", "WARNING", "IGNORED") } diff --git a/spec/models/import_service_spec.rb b/spec/models/import_service_spec.rb deleted file mode 100644 index e7ee062d6..000000000 --- a/spec/models/import_service_spec.rb +++ /dev/null @@ -1,19 +0,0 @@ -require 'spec_helper' - -describe ImportService, :type => :model do - - let(:referential) { create(:referential, :slug => "test") } - - subject { ImportService.new(referential) } - - describe '.find' do - - it "should build an import with a scheduled job" do - end - - it "should build an import with a terminated job" do - end - - end - -end diff --git a/spec/models/import_spec.rb b/spec/models/import_spec.rb index 8b85f151b..cbc5907fa 100644 --- a/spec/models/import_spec.rb +++ b/spec/models/import_spec.rb @@ -1,4 +1,4 @@ -RSpec.describe Import, type: :model do +RSpec.describe Import::Base, type: :model do it { should belong_to(:referential) } it { should belong_to(:workbench) } @@ -39,7 +39,7 @@ RSpec.describe Import, type: :model do ) current_import = create(:workbench_import, status: 'pending') - Import.abort_old + Import::Base.abort_old expect(current_import.reload.status).to eq('pending') expect(old_import.reload.status).to eq('aborted') @@ -54,7 +54,7 @@ RSpec.describe Import, type: :model do created_at: 4.hours.ago - 1.minute ) - Import.abort_old + Import::Base.abort_old expect(import.reload.status).to eq('successful') end @@ -73,7 +73,7 @@ RSpec.describe Import, type: :model do created_at: 4.hours.ago - 1.minute ) - NetexImport.abort_old + Import::Netex.abort_old expect(workbench_import.reload.status).to eq('pending') expect(netex_import.reload.status).to eq('aborted') @@ -88,25 +88,25 @@ RSpec.describe Import, type: :model do netex_import.parent.destroy expect(netex_import.parent).to be_destroyed - expect(NetexImport.count).to eq(0) + expect(Import::Netex.count).to eq(0) end - it "must destroy all associated ImportMessages" do + it "must destroy all associated Import::Messages" do import = create(:import) create(:import_resource, import: import) import.destroy - expect(ImportResource.count).to eq(0) + expect(Import::Resource.count).to eq(0) end - it "must destroy all associated ImportResources" do + it "must destroy all associated Import::Resources" do import = create(:import) create(:import_message, import: import) import.destroy - expect(ImportMessage.count).to eq(0) + expect(Import::Message.count).to eq(0) end end diff --git a/spec/models/netex_export_spec.rb b/spec/models/netex_export_spec.rb index 1d09fa07f..345bf4d5a 100644 --- a/spec/models/netex_export_spec.rb +++ b/spec/models/netex_export_spec.rb @@ -1,10 +1,10 @@ require 'spec_helper' -describe NetexExport, :type => :model do - - # describe '#export_options' do - # subject { super().export_options } - # it { is_expected.to include(:format => :netex) } - # end - -end +# describe NetexExport, :type => :model do +# +# # describe '#export_options' do +# # subject { super().export_options } +# # it { is_expected.to include(:format => :netex) } +# # end +# +# end diff --git a/spec/models/netex_import_spec.rb b/spec/models/netex_import_spec.rb deleted file mode 100644 index c6051a869..000000000 --- a/spec/models/netex_import_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -RSpec.describe NetexImport, type: :model do - describe "#destroy" do - it "must destroy its associated Referential if ready: false" do - workbench_import = create(:workbench_import) - referential_ready_false = create(:referential, ready: false) - referential_ready_true = create(:referential, ready: true) - create( - :netex_import, - parent: workbench_import, - referential: referential_ready_false - ) - create( - :netex_import, - parent: workbench_import, - referential: referential_ready_true - ) - - workbench_import.destroy - - expect( - Referential.where(id: referential_ready_false.id).exists? - ).to be false - expect( - Referential.where(id: referential_ready_true.id).exists? - ).to be true - end - - it "doesn't try to destroy nil referentials" do - workbench_import = create(:workbench_import) - create( - :netex_import, - parent: workbench_import, - referential: nil - ) - - expect { workbench_import.destroy }.not_to raise_error - end - end -end diff --git a/spec/policies/import_policy_spec.rb b/spec/policies/import_policy_spec.rb index 9c7fca8a5..f9effd21a 100644 --- a/spec/policies/import_policy_spec.rb +++ b/spec/policies/import_policy_spec.rb @@ -1,4 +1,4 @@ -RSpec.describe ImportPolicy, type: :policy do +RSpec.describe Import::BasePolicy, type: :policy do let( :record ){ build_stubbed :import } before { stub_policy_scope(record) } diff --git a/spec/requests/api/v1/netex_import_spec.rb b/spec/requests/api/v1/netex_import_spec.rb index 8597c1d32..14dac9a25 100644 --- a/spec/requests/api/v1/netex_import_spec.rb +++ b/spec/requests/api/v1/netex_import_spec.rb @@ -1,4 +1,4 @@ -RSpec.describe "NetexImport", type: :request do +RSpec.describe "Import::Netex", type: :request do describe 'POST netex_imports' do @@ -39,7 +39,7 @@ RSpec.describe "NetexImport", type: :request do post_request.(netex_import: legal_attributes) expect( response ).to be_success expect( json_response_body ).to eq( - 'id' => NetexImport.last.id, + 'id' => Import::Netex.last.id, 'referential_id' => Referential.last.id, 'workbench_id' => workbench.id ) @@ -51,7 +51,7 @@ RSpec.describe "NetexImport", type: :request do create(:line, objectid: 'STIF:CODIFLIGNE:Line:C00108', line_referential: workbench.line_referential) create(:line, objectid: 'STIF:CODIFLIGNE:Line:C00109', line_referential: workbench.line_referential) - expect{ post_request.(netex_import: legal_attributes) }.to change{NetexImport.count}.by(1) + expect{ post_request.(netex_import: legal_attributes) }.to change{Import::Netex.count}.by(1) end it 'creates a correct Referential', pending: 'see #5073' do @@ -96,7 +96,7 @@ RSpec.describe "NetexImport", type: :request do end it 'does not create an Import object' do - expect{ post_request.(netex_import: illegal_attributes) }.not_to change{Import.count} + expect{ post_request.(netex_import: illegal_attributes) }.not_to change{Import::Base.count} end it 'might not create a referential' do diff --git a/spec/services/parent_notifier_spec.rb b/spec/services/parent_notifier_spec.rb index ecf508fcd..d4ca73ab3 100644 --- a/spec/services/parent_notifier_spec.rb +++ b/spec/services/parent_notifier_spec.rb @@ -20,7 +20,7 @@ RSpec.describe ParentNotifier do expect(netex_import).to receive(:notify_parent) end - ParentNotifier.new(Import).notify_when_finished(netex_imports) + ParentNotifier.new(Import::Base).notify_when_finished(netex_imports) end it "doesn't call #notify_parent if its `notified_parent_at` is set" do @@ -33,7 +33,7 @@ RSpec.describe ParentNotifier do expect(netex_import).not_to receive(:notify_parent) - ParentNotifier.new(Import).notify_when_finished + ParentNotifier.new(Import::Base).notify_when_finished end end @@ -47,7 +47,7 @@ RSpec.describe ParentNotifier do ) expect( - ParentNotifier.new(Import).objects_pending_notification + ParentNotifier.new(Import::Base).objects_pending_notification ).to eq([netex_import]) end @@ -55,7 +55,7 @@ RSpec.describe ParentNotifier do create(:import, parent: nil) expect( - ParentNotifier.new(Import).objects_pending_notification + ParentNotifier.new(Import::Base).objects_pending_notification ).to be_empty end @@ -70,7 +70,7 @@ RSpec.describe ParentNotifier do end expect( - ParentNotifier.new(Import).objects_pending_notification + ParentNotifier.new(Import::Base).objects_pending_notification ).to be_empty end @@ -83,7 +83,7 @@ RSpec.describe ParentNotifier do ) expect( - ParentNotifier.new(Import).objects_pending_notification + ParentNotifier.new(Import::Base).objects_pending_notification ).to be_empty end end |