diff options
48 files changed, 1006 insertions, 91 deletions
diff --git a/app/concerns/configurable.rb b/app/concerns/configurable.rb new file mode 100644 index 000000000..c7d0f1fd9 --- /dev/null +++ b/app/concerns/configurable.rb @@ -0,0 +1,26 @@ +module Configurable + + module ClassMethods + def config &blk + blk ? blk.(configuration) : configuration + end + + private + def configuration + @__configuration__ ||= Rails::Application::Configuration.new + end + end + + module InstanceMethods + private + + def config + self.class.config + end + end + + def self.included(into) + into.extend ClassMethods + into.send :include, InstanceMethods + end +end diff --git a/app/controllers/api/v1/netex_imports_controller.rb b/app/controllers/api/v1/netex_imports_controller.rb index 16a7cef69..d67d121c0 100644 --- a/app/controllers/api/v1/netex_imports_controller.rb +++ b/app/controllers/api/v1/netex_imports_controller.rb @@ -19,7 +19,7 @@ module Api def netex_import_params params .require('netex_import') - .permit(:file, :name, :referential_id) + .permit(:file, :name, :referential_id, :workbench_id) end end end diff --git a/app/models/api/v1/api_key.rb b/app/models/api/v1/api_key.rb index 7390db232..e1a7ab5a4 100644 --- a/app/models/api/v1/api_key.rb +++ b/app/models/api/v1/api_key.rb @@ -3,9 +3,20 @@ module Api class ApiKey < ::ActiveRecord::Base before_create :generate_access_token belongs_to :referential, :class_name => '::Referential' + validates_presence_of :referential - def self.model_name - ActiveModel::Name.new self, Api::V1, self.name.demodulize + class << self + def from(referential, name:) + find_or_create_by!(name: name, referential: referential) + end + def model_name + ActiveModel::Name.new Api::V1, self.name.demodulize + end + def referential_from_token(token) + array = token.split('-') + return nil unless array.size==2 + ::Referential.find( array.first) + end end def eql?(other) @@ -13,16 +24,11 @@ module Api other.token == self.token end - def self.referential_from_token(token) - array = token.split('-') - return nil unless array.size==2 - ::Referential.find( array.first) - end private def generate_access_token begin - self.token = "#{referential.id}-#{SecureRandom.hex}" + self.token = "#{referential_id}-#{SecureRandom.hex}" end while self.class.exists?(:token => self.token) end end diff --git a/app/models/import.rb b/app/models/import.rb index 535c676b1..c932ecdd9 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -9,6 +9,7 @@ class Import < ActiveRecord::Base enumerize :status, in: %i(new pending successful failed running aborted canceled) validates :file, presence: true + validates_presence_of :referential, :workbench before_create do self.token_download = SecureRandom.urlsafe_base64 diff --git a/app/models/netex_import.rb b/app/models/netex_import.rb index 0cf4d0a7c..575cef816 100644 --- a/app/models/netex_import.rb +++ b/app/models/netex_import.rb @@ -2,6 +2,7 @@ require 'net/http' class NetexImport < Import after_commit :launch_java_import + def launch_java_import logger.warn "Call iev get #{Rails.configuration.iev_url}/boiv_iev/referentials/importer/new?id=#{id}" begin diff --git a/app/models/organisation.rb b/app/models/organisation.rb index d0742bda6..f697122aa 100644 --- a/app/models/organisation.rb +++ b/app/models/organisation.rb @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- class Organisation < ActiveRecord::Base include DataFormatEnumerations @@ -26,19 +25,12 @@ class Organisation < ActiveRecord::Base def self.portail_api_request conf = Rails.application.config.try(:stif_portail_api) - raise 'Rails.application.config.stif_portail_api settings is not defined' unless conf + raise 'Rails.application.config.stif_portail_api configuration is not defined' unless conf - conn = Faraday.new(:url => conf[:url]) do |c| - c.headers['Authorization'] = "Token token=\"#{conf[:key]}\"" - c.adapter Faraday.default_adapter - end - - resp = conn.get '/api/v1/organizations' - if resp.status == 200 - JSON.parse resp.body - else - raise "Error on api request status : #{resp.status} => #{resp.body}" - end + HTTPService.get_json_resource( + host: conf[:url], + path: '/api/v1/organizations', + token: conf[:key]) end def self.sync_update code, name, scope diff --git a/app/models/user.rb b/app/models/user.rb index c2aa14bda..37d35209a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -41,19 +41,12 @@ class User < ActiveRecord::Base def self.portail_api_request conf = Rails.application.config.try(:stif_portail_api) - raise 'Rails.application.config.stif_portail_api settings is not defined' unless conf + raise 'Rails.application.config.stif_portail_api configuration is not defined' unless conf - conn = Faraday.new(:url => conf[:url]) do |c| - c.headers['Authorization'] = %{Token token="#{conf[:key]}"} - c.adapter Faraday.default_adapter - end - - resp = conn.get '/api/v1/users' - if resp.status == 200 - JSON.parse resp.body - else - raise "Error on api request status : #{resp.status} => #{resp.body}" - end + HTTPService.get_json_resource( + host: conf[:url], + path: '/api/v1/users', + token: conf[:key]) end def self.portail_sync diff --git a/app/services/file_service.rb b/app/services/file_service.rb new file mode 100644 index 000000000..3b3ff3561 --- /dev/null +++ b/app/services/file_service.rb @@ -0,0 +1,24 @@ +# TODO: Delete me after stable implementation of #1726 +# module FileService extend self + +# def unique_filename( path, enum_with: with_ints ) +# file_names = enum_with.map( &file_name_maker(path) ) +# file_names +# .drop_while( &File.method(:exists?) ) +# .next +# end + +# def with_ints(format='%d') +# (0..Float::INFINITY) +# .lazy +# .map{ |n| format % n } +# end + + +# private + +# def file_name_maker path +# ->(n){ [path, n].join('_') } +# end + +# end diff --git a/app/services/http_service.rb b/app/services/http_service.rb new file mode 100644 index 000000000..ae7d0e413 --- /dev/null +++ b/app/services/http_service.rb @@ -0,0 +1,45 @@ +module HTTPService extend self + + Timeout = Faraday::TimeoutError + + def get_resource(host:, path:, token: nil, params: {}) + Faraday.new(url: host) do |c| + c.headers['Authorization'] = "Token token=#{token.inspect}" if token + c.adapter Faraday.default_adapter + + return c.get path, params + end + end + + def get_json_resource(host:, path:, token: nil, params: {}) + # Stupid Ruby!!! (I mean I just **need** Pattern Matching, maybe I need to write it myself :O) + resp = get_resource(host: host, path: path, token: token, params: params) + if resp.status == 200 + return JSON.parse(resp.body) + else + raise "Error on api request status : #{resp.status} => #{resp.body}" + end + end + + # host: 'http://localhost:3000', + # path: '/api/v1/netex_imports.json', + # token: '13-74009c36638f587c9eafb1ce46e95585', + # params: { netex_import: {referential_id: 13, workbench_id: 1}}, + # upload: {file: [StringIO.new('howdy'), 'application/zip', 'greeting']}) + def post_resource(host:, path:, token: nil, params: {}, upload: nil) + Faraday.new(url: host) do |c| + c.headers['Authorization'] = "Token token=#{token.inspect}" if token + c.request :multipart + c.request :url_encoded + c.adapter Faraday.default_adapter + + if upload + name = upload.keys.first + value, mime_type, as_name = upload.values.first + params.update( name => Faraday::UploadIO.new(value, mime_type, as_name ) ) + end + + return c.post path, params + end + end +end diff --git a/app/services/retry_service.rb b/app/services/retry_service.rb new file mode 100644 index 000000000..21b1def36 --- /dev/null +++ b/app/services/retry_service.rb @@ -0,0 +1,54 @@ +require 'result' + +class RetryService + + Retry = Class.new(RuntimeError) + + # @param@ delays: + # An array of delays that are used to retry after a sleep of the indicated + # value in case of failed exceutions. + # Once this array is exhausted the executen fails permanently + # + # @param@ rescue_from: + # During execution all the excpetions from this array +plus RetryService::Retry+ are rescued from and + # trigger just another retry after a `sleep` as indicated above. + # + # @param@ block: + # This optional code is excuted before each retry, it is passed the result of the failed attempt, thus + # an `Exception` and the number of execution already tried. + def initialize( delays: [], rescue_from: [], &blk ) + @intervals = delays + @registered_exceptions = Array(rescue_from) << Retry + @failure_callback = blk + end + + # @param@ blk: + # The code to be executed it will be retried goverened by the `delay` passed into the initializer + # as described there in case it fails with one of the predefined exceptions or `RetryService::Retry` + # + # Eventually it will return a `Result` object. + def execute &blk + result = execute_protected blk + return result if result.ok? + @intervals.each_with_index do | interval, retry_count | + sleep interval + @failure_callback.try(:call, result.value, retry_count + 1) + result = execute_protected blk + return result if result.ok? + end + result + end + + + private + + def execute_protected blk + Result.ok(blk.()) + rescue Exception => e + if @registered_exceptions.any?{ |re| e.is_a? re } + Result.error(e) + else + raise + end + end +end diff --git a/app/services/zip_service.rb b/app/services/zip_service.rb new file mode 100644 index 000000000..778bfd06d --- /dev/null +++ b/app/services/zip_service.rb @@ -0,0 +1,55 @@ +class ZipService + + attr_reader :current_entry, :zip_data + + def initialize data + @zip_data = data + @current_entry = nil + end + + class << self + def convert_entries entries + -> output_stream do + entries.each do |e| + output_stream.put_next_entry e.name + output_stream.write e.get_input_stream.read + end + end + end + + def entries input_stream + Enumerator.new do |enum| + loop{ enum << input_stream.get_next_entry } + end.lazy.take_while{ |e| e } + end + end + + def entry_groups + self.class.entries(input_stream).group_by(&method(:entry_key)) + end + + def entry_group_streams + entry_groups.map(&method(:make_stream)).to_h + end + + def entry_key entry + entry.name.split('/', -1)[-2] + end + + def make_stream pair + name, entries = pair + [name, make_stream_from( entries )] + end + + def make_stream_from entries + Zip::OutputStream.write_buffer(&self.class.convert_entries(entries)) + end + + def next_entry + @current_entry = input_stream.get_next_entry + end + + def input_stream + @__input_stream__ ||= Zip::InputStream.open(StringIO.new(zip_data)) + end +end diff --git a/app/workers/workbench_import_worker.rb b/app/workers/workbench_import_worker.rb new file mode 100644 index 000000000..57c0f5f4e --- /dev/null +++ b/app/workers/workbench_import_worker.rb @@ -0,0 +1,118 @@ +class WorkbenchImportWorker + include Sidekiq::Worker + include Rails.application.routes.url_helpers + include Configurable + + RETRY_DELAYS = [3, 5, 8] + + # Workers + # ======= + + def perform(import_id) + @import = WorkbenchImport.find(import_id) + @response = nil + @import.update_attributes(status: 'running') + downloaded = download + zip_service = ZipService.new(downloaded) + upload zip_service + end + + def download + logger.info "HTTP GET #{import_url}" + @zipfile_data = HTTPService.get_resource( + host: import_host, + path: import_path, + params: {token: @import.token_download}).body + end + + def execute_post eg_name, eg_stream + logger.info "HTTP POST #{export_url} (for #{complete_entry_group_name(eg_name)})" + HTTPService.post_resource( + host: export_host, + path: export_path, + token: token(eg_name), + params: params, + upload: {file: [eg_stream, 'application/zip', eg_name]}) + end + + def log_failure reason, count + logger.warn "HTTP POST failed with #{reason}, count = #{count}, response=#{@response}" + end + + def try_again + raise RetryService::Retry + end + + def try_upload_entry_group eg_name, eg_stream + result = execute_post eg_name, eg_stream + return result if result && result.status < 400 + @response = result.body + try_again + end + + def upload zip_service + entry_group_streams = zip_service.entry_group_streams + @import.update_attributes total_steps: entry_group_streams.size + entry_group_streams.each_with_index(&method(:upload_entry_group)) + rescue StopIteration + @import.update_attributes( current_step: entry_group_streams.size, status: 'failed' ) + end + + def upload_entry_group key_pair, element_count + @import.update_attributes( current_step: element_count.succ ) + retry_service = RetryService.new( + delays: RETRY_DELAYS, + rescue_from: [HTTPService::Timeout], + &method(:log_failure)) + status = retry_service.execute(&upload_entry_group_proc(key_pair)) + raise StopIteration unless status.ok? + end + + def upload_entry_group_proc key_pair + eg_name, eg_stream = key_pair + # This should be fn.try_upload_entry_group(eg_name, eg_stream) ;( + -> do + try_upload_entry_group(eg_name, eg_stream) + end + end + + + + # Queries + # ======= + + def complete_entry_group_name entry_group_name + [@import.name, entry_group_name].join("--") + end + + def token entry_group_name + Api::V1::ApiKey.from(@import.referential, name: complete_entry_group_name(entry_group_name)).token + end + + # Constants + # ========= + + def export_host + Rails.application.config.rails_host + end + def export_path + api_v1_netex_imports_path(format: :json) + end + def export_url + @__export_url__ ||= File.join(export_host, export_path) + end + + def import_host + Rails.application.config.rails_host + end + def import_path + @__import_path__ ||= download_workbench_import_path(@import.workbench, @import) + end + def import_url + @__import_url__ ||= File.join(import_host, import_path) + end + + def params + @__params__ ||= { netex_import: { referential_id: @import.referential_id, workbench_id: @import.workbench_id } } + end +end diff --git a/config/application.rb b/config/application.rb index 910ddd983..05a9752b6 100644 --- a/config/application.rb +++ b/config/application.rb @@ -14,7 +14,7 @@ module ChouetteIhm # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers # -- all .rb files in that directory are automatically loaded. - config.autoload_paths << config.root.join("lib") + config.autoload_paths << config.root.join('lib') # custom exception pages config.exceptions_app = self.routes diff --git a/config/environments/development.rb b/config/environments/development.rb index 5b2bd7402..ab4a29a3e 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -82,7 +82,8 @@ Rails.application.configure do config.portal_url = "http://stif-boiv-staging.af83.priv" # IEV url - config.iev_url = "http://localhost:8080" + config.iev_url = ENV.fetch('IEV_URL', 'http://localhost:8080') + config.rails_host = ENV.fetch('FRONT_END_URL', 'http://localhost:3000') # file to data for demo config.demo_data = "tmp/demo.zip" diff --git a/config/environments/test.rb b/config/environments/test.rb index a6db12006..b3312be4a 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -62,6 +62,7 @@ Rails.application.configure do # Reflex api url config.reflex_api_url = "https://195.46.215.128/ws/reflex/V1/service=getData" + config.rails_host = "http://www.example.com" # file to data for demo config.demo_data = "tmp/demo.zip" diff --git a/config/initializers/apartment.rb b/config/initializers/apartment.rb index 29ce6564f..e1e86449c 100644 --- a/config/initializers/apartment.rb +++ b/config/initializers/apartment.rb @@ -18,34 +18,35 @@ Apartment.configure do |config| # config.excluded_models = %w{Tenant} # config.excluded_models = [ - "Referential", - "ReferentialMetadata", - "Organisation", - "User", - "Api::V1::ApiKey", - "RuleParameterSet", - "StopAreaReferential", - "StopAreaReferentialMembership", - "StopAreaReferentialSync", - "StopAreaReferentialSyncMessage", - "Chouette::StopArea", - "LineReferential", - "LineReferentialMembership", - "LineReferentialSync", - "LineReferentialSyncMessage", - "Chouette::Line", - "Chouette::GroupOfLine", - "Chouette::Company", - "Chouette::Network", - "ReferentialCloning", - "Workbench", - "CleanUp", - "CleanUpResult", - "Calendar", - "Import", - "NetexImport", - "ImportMessage", - "ImportResource" + 'Referential', + 'ReferentialMetadata', + 'Organisation', + 'User', + 'Api::V1::ApiKey', + 'RuleParameterSet', + 'StopAreaReferential', + 'StopAreaReferentialMembership', + 'StopAreaReferentialSync', + 'StopAreaReferentialSyncMessage', + 'Chouette::StopArea', + 'LineReferential', + 'LineReferentialMembership', + 'LineReferentialSync', + 'LineReferentialSyncMessage', + 'Chouette::Line', + 'Chouette::GroupOfLine', + 'Chouette::Company', + 'Chouette::Network', + 'ReferentialCloning', + 'Workbench', + 'CleanUp', + 'CleanUpResult', + 'Calendar', + 'Import', + 'NetexImport', + 'WorkbenchImport', + 'ImportMessage', + 'ImportResource' ] # use postgres schemas? diff --git a/config/initializers/workbench_import.rb b/config/initializers/workbench_import.rb new file mode 100644 index 000000000..89ddd72ef --- /dev/null +++ b/config/initializers/workbench_import.rb @@ -0,0 +1,5 @@ +WorkbenchImportWorker.config do | config | + config.dir = ENV.fetch('WORKBENCH_IMPORT_DIR'){ Rails.root.join 'tmp/workbench_import' } + + FileUtils.mkdir_p config.dir +end diff --git a/db/migrate/20170727130705_add_current_step_and_total_steps_to_import.rb b/db/migrate/20170727130705_add_current_step_and_total_steps_to_import.rb new file mode 100644 index 000000000..b31e86f17 --- /dev/null +++ b/db/migrate/20170727130705_add_current_step_and_total_steps_to_import.rb @@ -0,0 +1,6 @@ +class AddCurrentStepAndTotalStepsToImport < ActiveRecord::Migration + def change + add_column :imports, :current_step, :integer, default: 0 + add_column :imports, :total_steps, :integer, default: 0 + end +end diff --git a/db/schema.rb b/db/schema.rb index b56027f48..05a024e1d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170715041954) do +ActiveRecord::Schema.define(version: 20170727130705) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -287,6 +287,8 @@ ActiveRecord::Schema.define(version: 20170715041954) do t.string "type", limit: 255 t.integer "parent_id", limit: 8 t.string "parent_type" + t.integer "current_step", default: 0 + t.integer "total_steps", default: 0 end add_index "imports", ["referential_id"], name: "index_imports_on_referential_id", using: :btree diff --git a/lib/result.rb b/lib/result.rb new file mode 100644 index 000000000..96e03d323 --- /dev/null +++ b/lib/result.rb @@ -0,0 +1,37 @@ +# A value wrapper adding status information to any value +# Status can be :ok or :error, we are thusly implementing +# what is expressed in Elixir/Erlang as result tuples and +# in Haskell as `Data.Either` +class Result + + attr_reader :status, :value + + class << self + def ok value + make :ok, value + end + def error value + make :error, value + end + + def new *args + raise NoMethodError, "No default constructor for #{self}" + end + + private + def make status, value + allocate.tap do | o | + o.instance_exec do + @status = status + @value = value + end + end + end + end + + def ok?; status == :ok end + + def == other + other.kind_of?(self.class) && other.status == status && other.value == value + end +end diff --git a/spec/concerns/configurable_spec.rb b/spec/concerns/configurable_spec.rb new file mode 100644 index 000000000..330241b72 --- /dev/null +++ b/spec/concerns/configurable_spec.rb @@ -0,0 +1,35 @@ +RSpec.describe Configurable do + + subject do + Class.new do + include Configurable + end + end + + let( :something ){ double('something') } + + it 'can be configured' do + expect{ subject.config.anything }.to raise_error(NoMethodError) + + subject.config.something = something + + expect( subject.config.something ).to eq(something) + # Instances delegate to the class + expect( subject.new.send(:config).something ).to eq(something) + # **All** instances delegate to the class + expect( subject.new.send(:config).something ).to eq(something) + end + + it 'can be configured with a block' do + + subject.config do | c | + c.something = something + end + + expect( subject.config.something ).to eq(something) + # Instances delegate to the class + expect( subject.new.send(:config).something ).to eq(something) + # **All** instances delegate to the class + expect( subject.new.send(:config).something ).to eq(something) + end +end diff --git a/spec/controllers/imports_controller_spec.rb b/spec/controllers/imports_controller_spec.rb index 7b575ab61..f07190496 100644 --- a/spec/controllers/imports_controller_spec.rb +++ b/spec/controllers/imports_controller_spec.rb @@ -15,6 +15,7 @@ RSpec.describe ImportsController, :type => :controller do it 'should be successful' do get :download, workbench_id: workbench.id, id: import.id, token: import.token_download expect(response).to be_success + expect( response.body ).to eq(import.file.read) end end end diff --git a/spec/factories/api_keys.rb b/spec/factories/api_keys.rb new file mode 100644 index 000000000..bd31edecc --- /dev/null +++ b/spec/factories/api_keys.rb @@ -0,0 +1,6 @@ +FactoryGirl.define do + factory :api_key, class: Api::V1::ApiKey do + token { "#{referential.id}-#{SecureRandom.hex}" } + referential + end +end diff --git a/spec/factories/import_tasks.rb b/spec/factories/import_tasks.rb deleted file mode 100644 index 9ca6db899..000000000 --- a/spec/factories/import_tasks.rb +++ /dev/null @@ -1,10 +0,0 @@ -FactoryGirl.define do - factory :import_task do |f| - user_name "dummy" - user_id 123 - no_save false - format "Neptune" - resources { Rack::Test::UploadedFile.new 'spec/fixtures/neptune.zip', 'application/zip', false } - referential { Referential.find_by_slug("first") } - end -end diff --git a/spec/factories/imports.rb b/spec/factories/imports.rb index e19fe92bb..aa9288fe9 100644 --- a/spec/factories/imports.rb +++ b/spec/factories/imports.rb @@ -13,5 +13,8 @@ FactoryGirl.define do factory :netex_import, class: NetexImport do file {File.open(Rails.root.join('spec', 'fixtures', 'terminated_job.json'))} end + factory :workbench_import, class: WorkbenchImport do + file {File.open(Rails.root.join('spec', 'fixtures', 'terminated_job.json'))} + end end end diff --git a/spec/fixtures/multiple_references_import.zip b/spec/fixtures/multiple_references_import.zip Binary files differnew file mode 100644 index 000000000..28ddff198 --- /dev/null +++ b/spec/fixtures/multiple_references_import.zip diff --git a/spec/fixtures/neptune.zip b/spec/fixtures/neptune.zip Binary files differdeleted file mode 100644 index 86b688b51..000000000 --- a/spec/fixtures/neptune.zip +++ /dev/null diff --git a/spec/fixtures/nozip.zip b/spec/fixtures/nozip.zip new file mode 100644 index 000000000..505bd213a --- /dev/null +++ b/spec/fixtures/nozip.zip @@ -0,0 +1 @@ +no zip file diff --git a/spec/fixtures/ref1.zip b/spec/fixtures/ref1.zip Binary files differnew file mode 100644 index 000000000..1cbd0268e --- /dev/null +++ b/spec/fixtures/ref1.zip diff --git a/spec/fixtures/ref2.zip b/spec/fixtures/ref2.zip Binary files differnew file mode 100644 index 000000000..342353b07 --- /dev/null +++ b/spec/fixtures/ref2.zip diff --git a/spec/fixtures/single_reference_import.zip b/spec/fixtures/single_reference_import.zip Binary files differnew file mode 100644 index 000000000..4aee23614 --- /dev/null +++ b/spec/fixtures/single_reference_import.zip diff --git a/spec/lib/result_spec.rb b/spec/lib/result_spec.rb new file mode 100644 index 000000000..949de163c --- /dev/null +++ b/spec/lib/result_spec.rb @@ -0,0 +1,20 @@ +RSpec.describe Result do + + context 'is a wrapper of a value' do + it { expect( described_class.ok('hello').value ).to eq('hello') } + it { expect( described_class.error('hello').value ).to eq('hello') } + end + + context 'it has status information' do + it { expect( described_class.ok('hello') ).to be_ok } + it { expect( described_class.ok('hello').status ).to eq(:ok) } + + it { expect( described_class.error('hello') ).not_to be_ok } + it { expect( described_class.error('hello').status ).to eq(:error) } + end + + context 'nil is just another value' do + it { expect( described_class.ok(nil) ).to be_ok } + it { expect( described_class.ok(nil).value ).to be_nil } + end +end diff --git a/spec/models/api/v1/api_key_spec.rb b/spec/models/api/v1/api_key_spec.rb index 8a34c9221..5f39a65e4 100644 --- a/spec/models/api/v1/api_key_spec.rb +++ b/spec/models/api/v1/api_key_spec.rb @@ -1,11 +1,34 @@ describe Api::V1::ApiKey, :type => :model do - let!(:referential){create(:referential)} - subject { Api::V1::ApiKey.create( :name => "test", :referential => referential)} - it "test" do - expect(subject).to be_valid + let(:referential){ create :referential } + + subject { described_class.create( :name => "test", :referential => referential)} + + it "validity test" do + expect_it.to be_valid expect(subject.referential).to eq(referential) + end + + context 'Creation' do + let( :name ){ SecureRandom.urlsafe_base64 } + + it 'can be created from a referential with a name, iff needed' do + # 1st time create a new record + expect{ described_class.from(referential, name: name) }.to change{ described_class.count }.by(1) + expect( described_class.last.attributes.values_at(*%w{referential_id name}) ).to eq([ + referential.id, name + ]) + + # 2nd time get the same record + expect{ described_class.from(referential, name: name) }.not_to change{ described_class.count } + expect( described_class.last.attributes.values_at(*%w{referential_id name}) ).to eq([ + referential.id, name + ]) + end + it 'cannot be created without a referential' do + expect{ described_class.from(nil, name:name) rescue nil }.not_to change{ described_class.count } + end end end diff --git a/spec/models/import_spec.rb b/spec/models/import_spec.rb index 34bfb0b23..c56858b44 100644 --- a/spec/models/import_spec.rb +++ b/spec/models/import_spec.rb @@ -1,11 +1,12 @@ -require 'rails_helper' - RSpec.describe Import, :type => :model do + it { should belong_to(:referential) } it { should belong_to(:workbench) } it { should belong_to(:parent).class_name(described_class.to_s) } - it { should enumerize(:status).in(:new, :pending, :successful, :failed, :canceled, :running, :aborted ) } + it { should enumerize(:status).in("aborted", "canceled", "failed", "new", "pending", "running", "successful") } it { should validate_presence_of(:file) } + it { should validate_presence_of(:referential) } + it { should validate_presence_of(:workbench) } end diff --git a/spec/models/organisation_spec.rb b/spec/models/organisation_spec.rb index 527f71015..b16324a56 100644 --- a/spec/models/organisation_spec.rb +++ b/spec/models/organisation_spec.rb @@ -1,5 +1,3 @@ -require 'spec_helper' - describe Organisation, :type => :model do it { should validate_presence_of(:name) } it { should validate_uniqueness_of(:code) } @@ -17,7 +15,7 @@ describe Organisation, :type => :model do let(:conf) { Rails.application.config.stif_portail_api } before :each do stub_request(:get, "#{conf[:url]}/api/v1/organizations"). - with(headers: { 'Authorization' => "Token token=\"#{conf[:key]}\"" }). + with(stub_headers(authorization_token: conf[:key])). to_return(body: File.open(File.join(Rails.root, 'spec', 'fixtures', 'organizations.json')), status: 200) end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 3a9ae37e9..51ccfccd3 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -67,7 +67,7 @@ RSpec.describe User, :type => :model do let(:conf) { Rails.application.config.stif_portail_api } before :each do stub_request(:get, "#{conf[:url]}/api/v1/users"). - with(headers: { 'Authorization' => "Token token=\"#{conf[:key]}\"" }). + with(stub_headers(authorization_token: conf[:key])). to_return(body: File.open(File.join(Rails.root, 'spec', 'fixtures', 'users.json')), status: 200) end diff --git a/spec/requests/api/v1/netex_import_spec.rb b/spec/requests/api/v1/netex_import_spec.rb index 9fbf8f801..ab1e7f6ae 100644 --- a/spec/requests/api/v1/netex_import_spec.rb +++ b/spec/requests/api/v1/netex_import_spec.rb @@ -4,7 +4,7 @@ RSpec.describe "NetexImport", type: :request do let( :referential ){ create :referential } - let( :file_path ){'spec/fixtures/neptune.zip'} + let( :file_path ){ fixtures_path 'single_reference_import.zip' } let( :file ){ fixture_file_upload( file_path ) } let( :post_request ) do @@ -19,7 +19,8 @@ RSpec.describe "NetexImport", type: :request do { name: 'hello world', file: file, - referential_id: referential.id + referential_id: referential.id, + workbench_id: referential.workbench_id } end diff --git a/spec/services/file_service_spec.rb b/spec/services/file_service_spec.rb new file mode 100644 index 000000000..4426ee145 --- /dev/null +++ b/spec/services/file_service_spec.rb @@ -0,0 +1,17 @@ +# TODO: Delete me after stable implementation of #1726 +# RSpec.describe FileService do + +# it 'computes a unique filename' do +# expect( File ).to receive(:exists?).with('xxx/yyy_0').and_return( false ) + +# expect(described_class.unique_filename('xxx/yyy')).to eq('xxx/yyy_0') +# end + +# it 'handles duplicate names by means of a counter' do +# expect( File ).to receive(:exists?).with('xxx/yyy_0').and_return( true ) +# expect( File ).to receive(:exists?).with('xxx/yyy_1').and_return( true ) +# expect( File ).to receive(:exists?).with('xxx/yyy_2').and_return( false ) + +# expect(described_class.unique_filename('xxx/yyy')).to eq('xxx/yyy_2') +# end +# end diff --git a/spec/services/http_service_spec.rb b/spec/services/http_service_spec.rb new file mode 100644 index 000000000..8c8af480c --- /dev/null +++ b/spec/services/http_service_spec.rb @@ -0,0 +1,74 @@ +RSpec.describe HTTPService do + + subject{ described_class } + + %i{host params path result}.each do |param| + let(param){ double(param) } + end + let( :token ){ SecureRandom.hex } + + let( :faraday_connection ){ double('faraday_connection') } + let( :headers ){ {} } + + + context 'get_resource' do + let( :params ){ double('params') } + + it 'sets authorization and returns result' do + expect(Faraday).to receive(:new).with(url: host).and_yield(faraday_connection) + expect(faraday_connection).to receive(:adapter).with(Faraday.default_adapter) + expect(faraday_connection).to receive(:headers).and_return headers + expect(faraday_connection).to receive(:get).with(path, params).and_return(result) + + expect(subject.get_resource(host: host, path: path, token: token, params: params)).to eq(result) + expect(headers['Authorization']).to eq( "Token token=#{token.inspect}" ) + end + end + + context 'post_resource' do + %i{as_name mime_type name upload_io value}.each do | param | + let( param ){ double(param) } + end + + let( :upload_list ){ [value, mime_type, as_name] } + + it 'sets authorization and posts data' do + expect(Faraday::UploadIO).to receive(:new).with(*upload_list).and_return upload_io + expect(params).to receive(:update).with(name => upload_io) + + expect(Faraday).to receive(:new).with(url: host).and_yield(faraday_connection) + expect(faraday_connection).to receive(:adapter).with(Faraday.default_adapter) + expect(faraday_connection).to receive(:headers).and_return headers + expect(faraday_connection).to receive(:request).with(:multipart) + expect(faraday_connection).to receive(:request).with(:url_encoded) + + expect(faraday_connection).to receive(:post).with(path, params).and_return(result) + + expect(subject.post_resource( + host: host, + path: path, + token: token, + params: params, + upload: {name => upload_list} )).to eq(result) + expect(headers['Authorization']).to eq( "Token token=#{token.inspect}" ) + end + + end + + context 'get_json_resource' do + + let( :content ){ SecureRandom.hex } + + it 'delegates an parses the response' do + expect_it.to receive(:get_resource) + .with(host: host, path: path, token: token, params: params) + .and_return(double(body: {content: content}.to_json, status: 200)) + + expect( subject.get_json_resource( + host: host, + path: path, + token: token, + params: params) ).to eq('content' => content) + end + end +end diff --git a/spec/services/retry_service_spec.rb b/spec/services/retry_service_spec.rb new file mode 100644 index 000000000..bb3416373 --- /dev/null +++ b/spec/services/retry_service_spec.rb @@ -0,0 +1,137 @@ +RSpec.describe RetryService do + subject { described_class.new delays: [2, 3], rescue_from: [NameError, ArgumentError] } + + context 'no retry necessary' do + before do + expect( subject ).not_to receive(:sleep) + end + + it 'returns an ok result' do + expect( subject.execute { 42 } ).to eq(Result.ok(42)) + end + it 'does not fail on nil' do + expect( subject.execute { nil } ).to eq(Result.ok(nil)) + end + + it 'fails wihout retries if raising un unregistered exception' do + expect{ subject.execute{ raise KeyError } }.to raise_error(KeyError) + end + + end + + context 'all retries fail' do + before do + expect( subject ).to receive(:sleep).with(2) + expect( subject ).to receive(:sleep).with(3) + end + it 'fails after raising a registered exception n times' do + result = subject.execute{ raise ArgumentError } + expect( result.status ).to eq(:error) + expect( result.value ).to be_kind_of(ArgumentError) + end + it 'fails with an explicit try again (automatically registered exception)' do + result = subject.execute{ raise RetryService::Retry } + expect( result.status ).to eq(:error) + expect( result.value ).to be_kind_of(RetryService::Retry) + end + end + + context "if at first you don't succeed" do + before do + @count = 0 + expect( subject ).to receive(:sleep).with(2) + end + + it 'succeeds the second time' do + expect( subject.execute{ succeed_later(ArgumentError){ 42 } } ).to eq(Result.ok(42)) + end + + it 'succeeds the second time with try again (automatically registered exception)' do + expect( subject.execute{ succeed_later(RetryService::Retry){ 42 } } ).to eq(Result.ok(42)) + end + end + + context 'last chance' do + before do + @count = 0 + expect( subject ).to receive(:sleep).with(2) + expect( subject ).to receive(:sleep).with(3) + end + it 'succeeds the third time with try again (automatically registered exception)' do + result = subject.execute{ succeed_later(RetryService::Retry, count: 2){ 42 } } + expect( result ).to eq( Result.ok(42) ) + end + end + + context 'failure callback once' do + subject do + described_class.new delays: [2, 3], rescue_from: [NameError, ArgumentError] do |reason, count| + @reason=reason + @callback_count=count + @failures += 1 + end + end + + before do + @failures = 0 + @count = 0 + expect( subject ).to receive(:sleep).with(2) + end + + it 'succeeds the second time and calls the failure_callback once' do + subject.execute{ succeed_later(RetryService::Retry){ 42 } } + expect( @failures ).to eq(1) + end + it '... and the failure is passed into the callback' do + subject.execute{ succeed_later(RetryService::Retry){ 42 } } + expect( @reason ).to be_a(RetryService::Retry) + expect( @callback_count ).to eq(1) + end + end + + context 'failure callback twice' do + subject do + described_class.new delays: [2, 3], rescue_from: [NameError, ArgumentError] do |_reason, _count| + @failures += 1 + end + end + + before do + @failures = 0 + @count = 0 + expect( subject ).to receive(:sleep).with(2) + expect( subject ).to receive(:sleep).with(3) + end + + it 'succeeds the third time and calls the failure_callback twice' do + subject.execute{ succeed_later(NameError, count: 2){ 42 } } + expect( @failures ).to eq(2) + end + end + + context 'failure callback in constructor' do + subject do + described_class.new(delays: [1, 2], &method(:add2failures)) + end + before do + @failures = [] + @count = 0 + expect( subject ).to receive(:sleep).with(1) + expect( subject ).to receive(:sleep).with(2) + end + it 'succeeds the second time and calls the failure_callback once' do + subject.execute{ succeed_later(RetryService::Retry, count: 2){ 42 } } + expect( @failures ).to eq([1,2]) + end + end + + def add2failures( e, c) + @failures << c + end + + def succeed_later error, count: 1, &blk + return blk.() unless @count < count + @count += 1 + raise error, 'error' + end +end diff --git a/spec/services/zip_service/zip_entry_data_spec.rb b/spec/services/zip_service/zip_entry_data_spec.rb new file mode 100644 index 000000000..2a7226eb4 --- /dev/null +++ b/spec/services/zip_service/zip_entry_data_spec.rb @@ -0,0 +1,32 @@ +RSpec.describe ZipService do + + subject{ described_class.new(read_fixture('multiple_references_import.zip')) } + + it 'can group all entries' do + expect( subject.entry_groups.keys ).to eq(%w{ref1 ref2}) + end + + context 'creates correct zip data for each subdir' do + it 'e.g. reference1' do + reference1_stream = subject.entry_group_streams['ref1'] + control_stream = Zip::InputStream.open( reference1_stream ) + control_entries = described_class.entries(control_stream) + expect( control_entries.map{ |e| [e.name, e.get_input_stream.read]}.force ).to eq([ + ["multiref/ref1/", ""], + ["multiref/ref1/datum-1", "multi-ref1-datum1\n"], + ["multiref/ref1/datum-2", "multi-ref1-datum2\n"] + ]) + end + it 'e.g. reference2' do + reference2_stream = subject.entry_group_streams['ref2'] + control_stream = Zip::InputStream.open( reference2_stream ) + control_entries = described_class.entries(control_stream) + expect( control_entries.map{ |e| [e.name, e.get_input_stream.read]}.force ).to eq([ + ["multiref/ref2/", ""], + ["multiref/ref2/datum-1", "multi-ref2-datum1\n"], + ["multiref/ref2/datum-2", "multi-ref2-datum2\n"] + ]) + end + end + +end diff --git a/spec/services/zip_service/zip_entry_dirs_spec.rb b/spec/services/zip_service/zip_entry_dirs_spec.rb new file mode 100644 index 000000000..8ca1b0f1a --- /dev/null +++ b/spec/services/zip_service/zip_entry_dirs_spec.rb @@ -0,0 +1,33 @@ +RSpec.describe ZipService do + + let( :zip_service ){ described_class } + + let( :zip_data ){ File.read zip_file } + + shared_examples_for 'a correct zip entry reader' do + it 'gets all entries of the zip file' do + expect( zip_service.new(zip_data).entry_groups.keys ).to eq(expected) + end + end + + context 'single entry' do + let( :zip_file ){ fixtures_path 'multiple_references_import.zip' } + let( :expected ){ %w{ref1 ref2} } + + it_behaves_like 'a correct zip entry reader' + end + + context 'more entries' do + let( :zip_file ){ fixtures_path 'single_reference_import.zip' } + let( :expected ){ %w{ref} } + + it_behaves_like 'a correct zip entry reader' + end + + context 'illegal file' do + let( :zip_file ){ fixtures_path 'nozip.zip' } + let( :expected ){ [] } + + it_behaves_like 'a correct zip entry reader' + end +end diff --git a/spec/services/zip_service/zip_output_streams_spec.rb b/spec/services/zip_service/zip_output_streams_spec.rb new file mode 100644 index 000000000..fbc60ae92 --- /dev/null +++ b/spec/services/zip_service/zip_output_streams_spec.rb @@ -0,0 +1,21 @@ +RSpec.describe ZipService do + + subject{ described_class.new(read_fixture('multiple_references_import.zip')) } + + + it 'can write itself to a file' do + streams = subject.entry_group_streams + streams.each do | name, stream | + File.write("tmp/#{name}.zip", stream.string) + end + ref1_lines = %x(unzip -l tmp/ref1.zip).split("\n").grep(%r{multiref/ref}).map(&:strip).map(&:split).map(&:last) + ref2_lines = %x(unzip -l tmp/ref2.zip).split("\n").grep(%r{multiref/ref}).map(&:strip).map(&:split).map(&:last) + + expect( ref1_lines ).to eq %w(multiref/ref1/ multiref/ref1/datum-1 multiref/ref1/datum-2) + expect( ref2_lines ).to eq %w(multiref/ref2/ multiref/ref2/datum-1 multiref/ref2/datum-2) + end + + it "exposes its size" do + expect( subject.entry_group_streams.size ).to eq(2) + end +end diff --git a/spec/support/fixtures_helper.rb b/spec/support/fixtures_helper.rb new file mode 100644 index 000000000..20963261b --- /dev/null +++ b/spec/support/fixtures_helper.rb @@ -0,0 +1,18 @@ +module Support + module FixturesHelper + def fixtures_path *segments + Rails.root.join( fixture_path, *segments ) + end + + def open_fixture *segments + File.open(fixtures_path(*segments)) + end + def read_fixture *segments + File.read(fixtures_path(*segments)) + end + end +end + +RSpec.configure do |c| + c.include Support::FixturesHelper +end diff --git a/spec/support/webmock/helpers.rb b/spec/support/webmock/helpers.rb new file mode 100644 index 000000000..fc6c77850 --- /dev/null +++ b/spec/support/webmock/helpers.rb @@ -0,0 +1,18 @@ +module Support + module Webmock + module Helpers + def stub_headers(*args) + {headers: make_headers(*args)} + end + + def make_headers(headers={}, authorization_token:) + headers.merge('Authorization' => "Token token=#{authorization_token.inspect}") + end + end + end +end + +RSpec.configure do | conf | + conf.include Support::Webmock::Helpers, type: :model + conf.include Support::Webmock::Helpers, type: :worker +end diff --git a/spec/tasks/reflex_rake_spec.rb b/spec/tasks/reflex_rake_spec.rb index 04c5886aa..6ece223d2 100644 --- a/spec/tasks/reflex_rake_spec.rb +++ b/spec/tasks/reflex_rake_spec.rb @@ -5,7 +5,7 @@ describe 'reflex:sync' do before(:each) do ['getOP', 'getOR'].each do |method| stub_request(:get, "#{Rails.application.config.reflex_api_url}/?format=xml&idRefa=0&method=#{method}"). - to_return(body: File.open("#{fixture_path}/reflex.zip"), status: 200) + to_return(body: open_fixture('reflex.zip'), status: 200) end stop_area_ref = create(:stop_area_referential, name: 'Reflex') @@ -43,7 +43,7 @@ describe 'reflex:sync' do before(:each) do ['getOP', 'getOR'].each do |method| stub_request(:get, "#{Rails.application.config.reflex_api_url}/?format=xml&idRefa=0&method=#{method}"). - to_return(body: File.open("#{fixture_path}/reflex_updated.zip"), status: 200) + to_return(body: open_fixture('reflex_updated.zip'), status: 200) end Stif::ReflexSynchronization.synchronize end diff --git a/spec/workers/stop_area_referential_sync_worker_spec.rb b/spec/workers/stop_area_referential_sync_worker_spec.rb index 48b64e55e..50c7cf45f 100644 --- a/spec/workers/stop_area_referential_sync_worker_spec.rb +++ b/spec/workers/stop_area_referential_sync_worker_spec.rb @@ -1,4 +1,3 @@ -require 'rails_helper' RSpec.describe StopAreaReferentialSyncWorker, type: :worker do let!(:stop_area_referential_sync) { create :stop_area_referential_sync } diff --git a/spec/workers/workbench_import_worker_spec.rb b/spec/workers/workbench_import_worker_spec.rb new file mode 100644 index 000000000..b719cbb98 --- /dev/null +++ b/spec/workers/workbench_import_worker_spec.rb @@ -0,0 +1,119 @@ +RSpec.describe WorkbenchImportWorker, type: [:worker, :request] do + + let( :worker ) { described_class.new } + let( :import ){ build_stubbed :import, token_download: download_token, file: zip_file } + + let( :workbench ){ import.workbench } + let( :referential ){ import.referential } + let( :api_key ){ build_stubbed :api_key, referential: referential, token: "#{referential.id}-#{SecureRandom.hex}" } + let( :params ) do + { netex_import: + { referential_id: referential.id, workbench_id: workbench.id } + } + end + + # http://www.example.com/workbenches/:workbench_id/imports/:id/download + let( :host ){ Rails.configuration.rails_host } + let( :path ){ download_workbench_import_path(workbench, import) } + + let( :downloaded_zip ){ double("downloaded zip") } + let( :download_zip_response ){ OpenStruct.new( body: downloaded_zip ) } + let( :download_token ){ SecureRandom.urlsafe_base64 } + + + let( :upload_path ) { api_v1_netex_imports_path(format: :json) } + + let( :entry_group_streams ) do + entry_count.times.map{ |i| double( "entry group stream #{i}" ) } + end + let( :entry_groups ) do + entry_count.times.map do | i | + {"entry_group_name#{i}" => entry_group_streams[i] } + end + end + + let( :zip_service ){ double("zip service") } + let( :zip_file ){ open_fixture('multiple_references_import.zip') } + + let( :post_response_ok ){ double(status: 201, body: "{}") } + + before do + # Silence Logger + allow_any_instance_of(Logger).to receive(:info) + allow_any_instance_of(Logger).to receive(:warn) + + # That should be `build_stubbed's` job, no? + allow(Import).to receive(:find).with(import.id).and_return(import) + + allow(Api::V1::ApiKey).to receive(:from).and_return(api_key) + allow(ZipService).to receive(:new).with(downloaded_zip).and_return zip_service + expect(zip_service).to receive(:entry_group_streams).and_return(entry_groups) + expect( import ).to receive(:update_attributes).with(status: 'running') + end + + + context 'multireferential zipfile, no errors' do + let( :entry_count ){ 2 } + + it 'downloads a zip file, cuts it, and uploads all pieces' do + + expect(HTTPService).to receive(:get_resource) + .with(host: host, path: path, params: {token: download_token}) + .and_return( download_zip_response ) + + entry_groups.each do | entry_group_name, entry_group_stream | + mock_post entry_group_name, entry_group_stream, post_response_ok + end + + expect( import ).to receive(:update_attributes).with(total_steps: 2) + expect( import ).to receive(:update_attributes).with(current_step: 1) + expect( import ).to receive(:update_attributes).with(current_step: 2) + + worker.perform import.id + + end + end + + context 'multireferential zipfile with error' do + let( :entry_count ){ 3 } + let( :post_response_failure ){ double(status: 406, body: {error: 'What was you thinking'}) } + + it 'downloads a zip file, cuts it, and uploads some pieces' do + expect(HTTPService).to receive(:get_resource) + .with(host: host, path: path, params: {token: download_token}) + .and_return( download_zip_response ) + + # First entry_group succeeds + entry_groups[0..0].each do | entry_group_name, entry_group_stream | + mock_post entry_group_name, entry_group_stream, post_response_ok + end + + # Second entry_group fails (M I S E R A B L Y) + entry_groups[1..1].each do | entry_group_name, entry_group_stream | + mock_post entry_group_name, entry_group_stream, post_response_failure + WorkbenchImportWorker::RETRY_DELAYS.each do | delay | + mock_post entry_group_name, entry_group_stream, post_response_failure + expect_any_instance_of(RetryService).to receive(:sleep).with(delay) + end + end + + expect( import ).to receive(:update_attributes).with(total_steps: 3) + expect( import ).to receive(:update_attributes).with(current_step: 1) + expect( import ).to receive(:update_attributes).with(current_step: 2) + expect( import ).to receive(:update_attributes).with(current_step: 3, status: 'failed') + + worker.perform import.id + + end + end + + def mock_post entry_group_name, entry_group_stream, response + expect( HTTPService ).to receive(:post_resource) + .with(host: host, + path: upload_path, + token: api_key.token, + params: params, + upload: {file: [entry_group_stream, 'application/zip', entry_group_name]}) + .and_return(response) + end +end |