From c1ac12174b9aff7535a84af9f76d1cda95b750f4 Mon Sep 17 00:00:00 2001 From: Zog Date: Fri, 2 Mar 2018 11:17:08 +0100 Subject: Refs #6068; Refactor import/export tasks --- lib/tasks/exports.rake | 47 ++++++++++++++++++ lib/tasks/helpers/simple_interfaces.rb | 27 +++++++++++ lib/tasks/imports.rake | 88 +++++++++------------------------- 3 files changed, 97 insertions(+), 65 deletions(-) create mode 100644 lib/tasks/exports.rake create mode 100644 lib/tasks/helpers/simple_interfaces.rb (limited to 'lib') diff --git a/lib/tasks/exports.rake b/lib/tasks/exports.rake new file mode 100644 index 000000000..036d96b11 --- /dev/null +++ b/lib/tasks/exports.rake @@ -0,0 +1,47 @@ +require 'csv' +require 'tasks/helpers/simple_interfaces' + +namespace :export do + desc "Notify parent imports when children finish" + task notify_parent: :environment do + ParentNotifier.new(Import).notify_when_finished + end + + desc "Mark old unfinished Netex imports as 'aborted'" + task netex_abort_old: :environment do + NetexImport.abort_old + end + + desc "export companies in the give LineReferential using the given exporter" + task :companies, [:referential_id, :configuration_name, :filepath, :logs_output_dir] => :environment do |t, args| + args.with_defaults(filepath: "./companies.csv", logs_output_dir: "./log/exporters/") + FileUtils.mkdir_p args[:logs_output_dir] + + referential = LineReferential.find args[:referential_id] + exporter = SimpleExporter.create configuration_name: args[:configuration_name], filepath: args[:filepath] + exporter.configure do |config| + config.collection = referential.companies.order(:name) + end + + SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args + end + + desc "export lines in the give LineReferential using the given exporter" + task :lines, [:referential_id, :configuration_name, :filepath, :logs_output_dir] => :environment do |t, args| + args.with_defaults(filepath: "./companies.csv", logs_output_dir: "./log/exporters/") + FileUtils.mkdir_p args[:logs_output_dir] + + referential = LineReferential.find args[:referential_id] + exporter = SimpleExporter.create configuration_name: args[:configuration_name], filepath: args[:filepath] + exporter.configure do |config| + config.collection = referential.lines.order(:name) + end + + SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args + end + + desc "export a complete offer from the gicen referential in the given X next days" + task :full_offer, [:referential_id, :timelapse, :configuration_name, :output_dir, :logs_output_dir] => :environment do |t, args| + args.with_defaults(filepath: "./companies.csv", logs_output_dir: "./log/exporters/") + end +end diff --git a/lib/tasks/helpers/simple_interfaces.rb b/lib/tasks/helpers/simple_interfaces.rb new file mode 100644 index 000000000..68e02e818 --- /dev/null +++ b/lib/tasks/helpers/simple_interfaces.rb @@ -0,0 +1,27 @@ +module SimpleInterfacesHelper + def self.interface_output_to_csv interface, output_dir + filepath = File.join output_dir, + "#{interface.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv" + cols = %w(line kind event message error) + if interface.reload.journal.size > 0 + keys = interface.journal.first["row"].map(&:first) + CSV.open(filepath, "w") do |csv| + csv << cols + keys + interface.journal.each do |j| + csv << cols.map{|c| j[c]} + j["row"].map(&:last) + end + end + puts "Task Output written in #{filepath}" + end + end + + def self.run_interface_controlling_interruption interface, method, args + begin + interface.send(method, verbose: true) + rescue Interrupt + raise + ensure + puts "\n\e[33m***\e[0m Done, status: " + (interface.status == "success" ? "\e[32m" : "\e[31m" ) + (interface.status || "") + "\e[0m" + interface_output_to_csv interface, args[:logs_output_dir] + end + end +end diff --git a/lib/tasks/imports.rake b/lib/tasks/imports.rake index f01d3f34f..cd9217e5a 100644 --- a/lib/tasks/imports.rake +++ b/lib/tasks/imports.rake @@ -1,4 +1,5 @@ require 'csv' +require 'tasks/helpers/simple_interfaces' namespace :import do desc "Notify parent imports when children finish" @@ -11,25 +12,10 @@ namespace :import do NetexImport.abort_old end - def importer_output_to_csv importer, output_dir - filepath = File.join output_dir, + "#{importer.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv" - cols = %w(line kind event message error) - if importer.reload.journal.size > 0 - keys = importer.journal.first["row"].map(&:first) - CSV.open(filepath, "w") do |csv| - csv << cols + keys - importer.journal.each do |j| - csv << cols.map{|c| j[c]} + j["row"].map(&:last) - end - end - puts "Import Output written in #{filepath}" - end - end - desc "import the given file with the corresponding importer" - task :import, [:configuration_name, :filepath, :referential_id, :output_dir] => :environment do |t, args| - args.with_defaults(output_dir: "./log/importers/") - FileUtils.mkdir_p args[:output_dir] + task :import, [:configuration_name, :filepath, :referential_id, :logs_output_dir] => :environment do |t, args| + args.with_defaults(logs_output_dir: "./log/importers/") + FileUtils.mkdir_p args[:logs_output_dir] importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath] @@ -37,46 +23,32 @@ namespace :import do referential = Referential.find args[:referential_id] importer.configure do |config| config.add_value :referential, referential - config.context = {referential: referential, output_dir: args[:output_dir]} + config.context = {referential: referential, logs_output_dir: args[:logs_output_dir]} end end - puts "\e[33m***\e[0m Start importing" - begin - importer.import(verbose: true) - rescue Interrupt - raise - ensure - puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m" - importer_output_to_csv importer, args[:output_dir] - end + + SimpleInterfacesHelper.run_interface_controlling_interruption importer, :import, args end desc "import the given file with the corresponding importer in the given StopAreaReferential" task :import_in_stop_area_referential, [:referential_id, :configuration_name, :filepath] => :environment do |t, args| - args.with_defaults(output_dir: "./log/importers/") - FileUtils.mkdir_p args[:output_dir] + args.with_defaults(logs_output_dir: "./log/importers/") + FileUtils.mkdir_p args[:logs_output_dir] referential = StopAreaReferential.find args[:referential_id] importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath] importer.configure do |config| config.add_value :stop_area_referential, referential - config.context = {stop_area_referential: referential, output_dir: args[:output_dir]} - end - puts "\e[33m***\e[0m Start importing" - begin - importer.import(verbose: true) - rescue Interrupt - raise - ensure - puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m" - importer_output_to_csv importer, args[:output_dir] + config.context = {stop_area_referential: referential, logs_output_dir: args[:logs_output_dir]} end + + SimpleInterfacesHelper.run_interface_controlling_interruption importer, :import, args end desc "import the given routes files" task :import_routes, [:referential_id, :configuration_name, :mapping_filepath, :filepath] => :environment do |t, args| - args.with_defaults(output_dir: "./log/importers/") - FileUtils.mkdir_p args[:output_dir] + args.with_defaults(logs_output_dir: "./log/importers/") + FileUtils.mkdir_p args[:logs_output_dir] referential = Referential.find args[:referential_id] referential.switch @@ -84,38 +56,24 @@ namespace :import do importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath] importer.configure do |config| config.add_value :stop_area_referential, referential - config.context = {stop_area_referential: stop_area_referential, mapping_filepath: args[:mapping_filepath], output_dir: args[:output_dir]} - end - puts "\e[33m***\e[0m Start importing" - begin - importer.import(verbose: true) - rescue Interrupt - raise - ensure - puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m" - importer_output_to_csv importer, args[:output_dir] + config.context = {stop_area_referential: stop_area_referential, mapping_filepath: args[:mapping_filepath], logs_output_dir: args[:logs_output_dir]} end + + SimpleInterfacesHelper.run_interface_controlling_interruption importer, :import, args end desc "import the given file with the corresponding importer in the given LineReferential" - task :import_in_line_referential, [:referential_id, :configuration_name, :filepath, :output_dir] => :environment do |t, args| - args.with_defaults(output_dir: "./log/importers/") - FileUtils.mkdir_p args[:output_dir] + task :import_in_line_referential, [:referential_id, :configuration_name, :filepath, :logs_output_dir] => :environment do |t, args| + args.with_defaults(logs_output_dir: "./log/importers/") + FileUtils.mkdir_p args[:logs_output_dir] referential = LineReferential.find args[:referential_id] importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath] importer.configure do |config| config.add_value :line_referential, referential - config.context = {line_referential: referential, output_dir: args[:output_dir]} - end - puts "\e[33m***\e[0m Start importing" - begin - importer.import(verbose: true) - rescue Interrupt - raise - ensure - puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m" - importer_output_to_csv importer, args[:output_dir] + config.context = {line_referential: referential, logs_output_dir: args[:logs_output_dir]} end + + SimpleInterfacesHelper.run_interface_controlling_interruption importer, :import, args end end -- cgit v1.2.3 From ea3de6035cdf9fbbcd92f51c90e5a2c5c3400cb3 Mon Sep 17 00:00:00 2001 From: Zog Date: Mon, 5 Mar 2018 09:11:34 +0100 Subject: Refs #6068; Export VehicleJourneys Add a mechanism to allow for several rows in the csv per single object in the collection. --- lib/tasks/exports.rake | 22 +++++++++++++++++++--- lib/tasks/helpers/simple_interfaces.rb | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/tasks/exports.rake b/lib/tasks/exports.rake index 036d96b11..6ff73dac0 100644 --- a/lib/tasks/exports.rake +++ b/lib/tasks/exports.rake @@ -40,8 +40,24 @@ namespace :export do SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args end - desc "export a complete offer from the gicen referential in the given X next days" - task :full_offer, [:referential_id, :timelapse, :configuration_name, :output_dir, :logs_output_dir] => :environment do |t, args| - args.with_defaults(filepath: "./companies.csv", logs_output_dir: "./log/exporters/") + desc "export a complete offer from the given referential in the given X next days" + task :full_offer, [:referential_id, :configuration_name, :timelapse, :output_dir, :logs_output_dir] => :environment do |t, args| + referential = Referential.find args[:referential_id] + args.with_defaults(output_dir: referential.name, logs_output_dir: "./log/exporters/", timelapse: 90) + + referential.switch + + journeys = Chouette::VehicleJourney.with_matching_timetable (Time.now.to_date..args[:timelapse].to_i.days.from_now.to_date) + if journeys.count == 0 + puts "No maching journeys were found".red + else + + exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_journeys", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_journeys.csv" + exporter.configure do |config| + config.collection = journeys + end + + SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args + end end end diff --git a/lib/tasks/helpers/simple_interfaces.rb b/lib/tasks/helpers/simple_interfaces.rb index 68e02e818..1dc051575 100644 --- a/lib/tasks/helpers/simple_interfaces.rb +++ b/lib/tasks/helpers/simple_interfaces.rb @@ -2,7 +2,7 @@ module SimpleInterfacesHelper def self.interface_output_to_csv interface, output_dir filepath = File.join output_dir, + "#{interface.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv" cols = %w(line kind event message error) - if interface.reload.journal.size > 0 + if interface.reload.journal.size > 0 && interface.journal.first["row"].present? keys = interface.journal.first["row"].map(&:first) CSV.open(filepath, "w") do |csv| csv << cols + keys -- cgit v1.2.3 From a412f915c885f3bf2962d0b786ff864f1b0e120e Mon Sep 17 00:00:00 2001 From: Zog Date: Mon, 5 Mar 2018 12:00:29 +0100 Subject: Refs #6068; Add some helpers in the models --- lib/tasks/exports.rake | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/tasks/exports.rake b/lib/tasks/exports.rake index 6ff73dac0..8688611e8 100644 --- a/lib/tasks/exports.rake +++ b/lib/tasks/exports.rake @@ -43,7 +43,7 @@ namespace :export do desc "export a complete offer from the given referential in the given X next days" task :full_offer, [:referential_id, :configuration_name, :timelapse, :output_dir, :logs_output_dir] => :environment do |t, args| referential = Referential.find args[:referential_id] - args.with_defaults(output_dir: referential.name, logs_output_dir: "./log/exporters/", timelapse: 90) + args.with_defaults(output_dir: "#{referential.name.parameterize}_#{Time.now.strftime "%y%m%d%H%M"}", logs_output_dir: "./log/exporters/", timelapse: 90) referential.switch @@ -51,12 +51,46 @@ namespace :export do if journeys.count == 0 puts "No maching journeys were found".red else - exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_journeys", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_journeys.csv" exporter.configure do |config| config.collection = journeys end + SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args + return unless exporter.status == :success + + exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_schedules", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_schedules.csv" + exporter.configure do |config| + config.collection = journeys + end + + SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args + return unless exporter.status == :success + + exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_routes", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_routes.csv" + exporter.configure do |config| + config.collection = Chouette::JourneyPattern.where(id: journeys.pluck(:journey_pattern_id).uniq) + end + + SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args + return unless exporter.status == :success + + exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_stops", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_stops.csv" + exporter.configure do |config| + config.collection = Chouette::StopArea.where(id: journeys.joins(:stop_points).pluck(:"stop_points.stop_area_id").uniq).order('parent_id ASC NULLS FIRST') + end + + SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args + return unless exporter.status == :success + + exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_companies", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_companies.csv" + ids = journeys.pluck :company_id + ids += journeys.joins(route: :line).pluck :"lines.company_id" + + exporter.configure do |config| + config.collection = Chouette::Company.where(id: ids.uniq).order('name') + end + SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args end end -- cgit v1.2.3 From 2b1a8cdc047cd80108ac4c5ef1fd2a64d77ab750 Mon Sep 17 00:00:00 2001 From: Zog Date: Mon, 5 Mar 2018 15:19:30 +0100 Subject: Refs #6068; Fully functional JSON exporter --- lib/tasks/exports.rake | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/tasks/exports.rake b/lib/tasks/exports.rake index 8688611e8..547388b35 100644 --- a/lib/tasks/exports.rake +++ b/lib/tasks/exports.rake @@ -43,7 +43,7 @@ namespace :export do desc "export a complete offer from the given referential in the given X next days" task :full_offer, [:referential_id, :configuration_name, :timelapse, :output_dir, :logs_output_dir] => :environment do |t, args| referential = Referential.find args[:referential_id] - args.with_defaults(output_dir: "#{referential.name.parameterize}_#{Time.now.strftime "%y%m%d%H%M"}", logs_output_dir: "./log/exporters/", timelapse: 90) + args.with_defaults(output_dir: "#{referential.name.parameterize}/#{Time.now.strftime "%y%m%d%H%M"}", logs_output_dir: "./log/exporters/", timelapse: 90) referential.switch @@ -51,44 +51,44 @@ namespace :export do if journeys.count == 0 puts "No maching journeys were found".red else - exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_journeys", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_journeys.csv" + exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_companies", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_companies.json" + ids = journeys.pluck :company_id + ids += journeys.joins(route: :line).pluck :"lines.company_id" + exporter.configure do |config| - config.collection = journeys + config.collection = Chouette::Company.where(id: ids.uniq).order('name') end SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args - return unless exporter.status == :success + break if exporter.status == :error - exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_schedules", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_schedules.csv" + exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_schedules", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_schedules.json" exporter.configure do |config| config.collection = journeys end SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args - return unless exporter.status == :success + break if exporter.status == :error - exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_routes", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_routes.csv" + exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_routes", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_routes.json" exporter.configure do |config| config.collection = Chouette::JourneyPattern.where(id: journeys.pluck(:journey_pattern_id).uniq) end SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args - return unless exporter.status == :success + break if exporter.status == :error - exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_stops", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_stops.csv" + exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_stops", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_stops.json" exporter.configure do |config| config.collection = Chouette::StopArea.where(id: journeys.joins(:stop_points).pluck(:"stop_points.stop_area_id").uniq).order('parent_id ASC NULLS FIRST') end SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args - return unless exporter.status == :success - - exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_companies", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_companies.csv" - ids = journeys.pluck :company_id - ids += journeys.joins(route: :line).pluck :"lines.company_id" + break if exporter.status == :error + exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_journeys", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_journeys.json" exporter.configure do |config| - config.collection = Chouette::Company.where(id: ids.uniq).order('name') + config.collection = journeys end SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args -- cgit v1.2.3 From dd11d9c45ea1ceb473f450224ebf6345922a7325 Mon Sep 17 00:00:00 2001 From: Alban Peignier Date: Mon, 5 Mar 2018 22:42:54 +0100 Subject: Create output_dir if needed. Refs #6068 --- lib/tasks/helpers/simple_interfaces.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/tasks/helpers/simple_interfaces.rb b/lib/tasks/helpers/simple_interfaces.rb index 1dc051575..5b593be43 100644 --- a/lib/tasks/helpers/simple_interfaces.rb +++ b/lib/tasks/helpers/simple_interfaces.rb @@ -1,5 +1,6 @@ module SimpleInterfacesHelper def self.interface_output_to_csv interface, output_dir + FileUtils.mkdir_p output_dir filepath = File.join output_dir, + "#{interface.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv" cols = %w(line kind event message error) if interface.reload.journal.size > 0 && interface.journal.first["row"].present? -- cgit v1.2.3