From fc53a6c6c1fb95ce46e13f501344d71a6d1e6de9 Mon Sep 17 00:00:00 2001 From: Alban Peignier Date: Fri, 30 Mar 2018 09:07:33 +0200 Subject: Initial import for agencies, stops, routes, trips and stop_times. Refs #6368 --- Gemfile | 2 + Gemfile.lock | 9 +- app/models/import/gtfs.rb | 160 ++++++++++++++++++++++++++---- app/models/referential.rb | 4 + app/workers/gtfs_import_worker.rb | 7 ++ spec/fixtures/google-sample-feed.zip | Bin 0 -> 3217 bytes spec/models/import/gtfs_spec.rb | 185 +++++++++++++++++++++++++++++++++++ 7 files changed, 344 insertions(+), 23 deletions(-) create mode 100644 app/workers/gtfs_import_worker.rb create mode 100644 spec/fixtures/google-sample-feed.zip create mode 100644 spec/models/import/gtfs_spec.rb diff --git a/Gemfile b/Gemfile index 6fcaf8896..5d5df4b06 100644 --- a/Gemfile +++ b/Gemfile @@ -146,6 +146,8 @@ gem 'puma', '~> 3.10.0' gem 'newrelic_rpm' gem 'letter_opener' +gem 'gtfs' + group :development do gem 'capistrano', '2.13.5' gem 'capistrano-ext' diff --git a/Gemfile.lock b/Gemfile.lock index 4e3c76690..7fd58c713 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -258,6 +258,10 @@ GEM google-analytics-rails (1.1.0) gretel (3.0.9) rails (>= 3.1.0) + gtfs (0.2.5) + multi_json + rake + rubyzip (~> 1.1) has_scope (0.7.0) actionpack (>= 4.1, < 5.1) activesupport (>= 4.1, < 5.1) @@ -321,7 +325,7 @@ GEM minitest (5.11.3) money (6.10.1) i18n (>= 0.6.4, < 1.0) - multi_json (1.12.1) + multi_json (1.13.1) multi_test (0.1.2) multi_xml (0.6.0) multipart-post (2.0.0) @@ -424,7 +428,7 @@ GEM thor (>= 0.18.1, < 2.0) rainbow (2.2.2) rake - rake (12.3.0) + rake (12.3.1) ransack (1.8.3) actionpack (>= 3.0) activerecord (>= 3.0) @@ -637,6 +641,7 @@ DEPENDENCIES georuby-ext (= 0.0.5) google-analytics-rails gretel + gtfs has_array_of! htmlbeautifier i18n-js diff --git a/app/models/import/gtfs.rb b/app/models/import/gtfs.rb index 03cf49e60..5b332315d 100644 --- a/app/models/import/gtfs.rb +++ b/app/models/import/gtfs.rb @@ -1,36 +1,154 @@ -require 'net/http' class Import::Gtfs < Import::Base - before_destroy :destroy_non_ready_referential + after_commit :launch_worker, :on => :create - after_commit :launch_java_import, on: :create - before_save def abort_unless_referential - self.status = 'aborted' unless referential + def launch_worker + GtfsImportWorker.perform_async id end - def launch_java_import - return if self.class.finished_statuses.include?(status) - threaded_call_boiv_iev + def import + workbench_import.update(status: 'running', started_at: Time.now) + + import_without_status + workbench_import.update(status: 'successful', ended_at: Time.now) + rescue Exception => e + workbench_import.update(status: 'failed', ended_at: Time.now) + raise e + end + + attr_accessor :local_file + + # TODO download the imported file + # def local_file + # @local_file + # end + + # TODO create referential with metadatas + # def referential + # ... + # end + + def source + @source ||= ::GTFS::Source.build local_file + end + + delegate :line_referential, :stop_area_referential, to: :referential + + def import_without_status + referential.switch + + import_agencies + import_stops + import_routes + import_trips + import_stop_times end - private + def import_agencies + source.agencies.each do |agency| + company = line_referential.companies.find_or_initialize_by(registration_number: agency.id) + company.attributes = { name: agency.name } - def destroy_non_ready_referential - if referential && !referential.ready - referential.destroy + save company end end - def threaded_call_boiv_iev - Thread.new(&method(:call_boiv_iev)) + def import_stops + source.stops.each do |stop| + stop_area = stop_area_referential.stop_areas.find_or_initialize_by(registration_number: stop.id) + + stop_area.name = stop.name + stop_area.area_type = stop.location_type == "1" ? "zdlp" : "zdep" + stop_area.parent = stop_referential.stop_areas.find_by!(registration_number: stop.parent_station) if stop.parent_station.present? + stop_area.latitude, stop_area.longitude = stop.lat, stop.lon + stop_area.kind = "commercial" + + # TODO correct default timezone + + save stop_area + end end - def call_boiv_iev - Rails.logger.error("Begin IEV call for import") - Net::HTTP.get(URI("#{Rails.configuration.iev_url}/boiv_iev/referentials/importer/new?id=#{id}")) - Rails.logger.error("End IEV call for import") - rescue Exception => e - logger.error "IEV server error : #{e.message}" - logger.error e.backtrace.inspect + def import_routes + source.routes.each do |route| + line = line_referential.lines.find_or_initialize_by(registration_number: route.id) + line.name = route.long_name.presence || route.short_name + line.number = route.short_name + line.published_name = route.long_name + + line.company = line_referential.companies.find_by(registration_number: route.agency_id) if route.agency_id.present? + + # TODO transport mode + + line.comment = route.desc + + # TODO colors + + line.url = route.url + + save line + end + end + + def vehicle_journey_by_trip_id + @vehicle_journey_by_trip_id ||= {} + end + + def import_trips + source.trips.each do |trip| + line = line_referential.lines.find_by registration_number: trip.route_id + + route = referential.routes.build line: line + route.wayback = (trip.direction_id == "0" ? :outbound : :inbound) + # TODO better name ? + name = route.published_name = trip.short_name.presence || trip.headsign.presence || route.wayback.to_s.capitalize + route.name = name + save route + + journey_pattern = route.journey_patterns.build name: name + save journey_pattern + + vehicle_journey = journey_pattern.vehicle_journeys.build route: route + vehicle_journey.published_journey_name = trip.headsign.presence || trip.id + save vehicle_journey + + vehicle_journey_by_trip_id[trip.id] = vehicle_journey.id + end + end + + def import_stop_times + source.stop_times.group_by(&:trip_id).each do |trip_id, stop_times| + vehicle_journey = referential.vehicle_journeys.find vehicle_journey_by_trip_id[trip_id] + journey_pattern = vehicle_journey.journey_pattern + route = journey_pattern.route + + stop_times.sort_by!(&:stop_sequence) + + stop_times.each do |stop_time| + stop_area = stop_area_referential.stop_areas.find_by(registration_number: stop_time.stop_id) + + stop_point = route.stop_points.build stop_area: stop_area + save stop_point + + journey_pattern.stop_points << stop_point + + # JourneyPattern#vjas_add creates automaticaly VehicleJourneyAtStop + vehicle_journey_at_stop = journey_pattern.vehicle_journey_at_stops.find_by(stop_point_id: stop_point.id) + vehicle_journey_at_stop.departure_time = stop_time.departure_time + vehicle_journey_at_stop.arrival_time = stop_time.arrival_time + + # TODO offset + + save vehicle_journey_at_stop + end + end + end + + def save(model) + unless model.save + Rails.logger.info "Can't save #{model.class.name} : #{model.errors.inspect}" + raise ActiveRecord::RecordNotSaved.new("Invalid #{model.class.name}") + end + Rails.logger.debug "Created #{model.inspect}" end end diff --git a/app/models/referential.rb b/app/models/referential.rb index 91a88d02d..65af58873 100644 --- a/app/models/referential.rb +++ b/app/models/referential.rb @@ -184,6 +184,10 @@ class Referential < ActiveRecord::Base Chouette::VehicleJourneyFrequency.all end + def vehicle_journey_at_stops + Chouette::VehicleJourneyAtStop.all + end + def routing_constraint_zones Chouette::RoutingConstraintZone.all end diff --git a/app/workers/gtfs_import_worker.rb b/app/workers/gtfs_import_worker.rb new file mode 100644 index 000000000..093869475 --- /dev/null +++ b/app/workers/gtfs_import_worker.rb @@ -0,0 +1,7 @@ +class WorkbenchImportWorker + include Sidekiq::Worker + + def perform(import_id) + Import::Gtfs.find(import_id).import + end +end diff --git a/spec/fixtures/google-sample-feed.zip b/spec/fixtures/google-sample-feed.zip new file mode 100644 index 000000000..79819e21a Binary files /dev/null and b/spec/fixtures/google-sample-feed.zip differ diff --git a/spec/models/import/gtfs_spec.rb b/spec/models/import/gtfs_spec.rb new file mode 100644 index 000000000..bdfc565d3 --- /dev/null +++ b/spec/models/import/gtfs_spec.rb @@ -0,0 +1,185 @@ +require "rails_helper" + +RSpec.describe Import::Gtfs do + + let(:referential) do + create :referential do |referential| + referential.line_referential.objectid_format = "netex" + referential.stop_area_referential.objectid_format = "netex" + end + end + + def create_import(file) + Import::Gtfs.new referential: referential, local_file: fixtures_path(file) + end + + describe "#import_agencies" do + let(:import) { create_import "google-sample-feed.zip" } + it "should create a company for each agency" do + import.import_agencies + + expect(referential.line_referential.companies.pluck(:registration_number, :name)).to eq([["DTA","Demo Transit Authority"]]) + end + end + + describe "#import_stops" do + let(:import) { create_import "google-sample-feed.zip" } + it "should create a company for each agency" do + import.import_stops + + defined_attributes = [ + :registration_number, :name, :parent_id, :latitude, :longitude + ] + expected_attributes = [ + ["AMV", "Amargosa Valley (Demo)", nil, 36.641496, -116.40094], + ["EMSI", "E Main St / S Irving St (Demo)", nil, 36.905697, -116.76218], + ["DADAN", "Doing Ave / D Ave N (Demo)", nil, 36.909489, -116.768242], + ["NANAA", "North Ave / N A Ave (Demo)", nil, 36.914944, -116.761472], + ["NADAV", "North Ave / D Ave N (Demo)", nil, 36.914893, -116.76821], + ["STAGECOACH", "Stagecoach Hotel & Casino (Demo)", nil, 36.915682, -116.751677], + ["BULLFROG", "Bullfrog (Demo)", nil, 36.88108, -116.81797], + ["BEATTY_AIRPORT", "Nye County Airport (Demo)", nil, 36.868446, -116.784582], + ["FUR_CREEK_RES", "Furnace Creek Resort (Demo)", nil, 36.425288, -117.133162] + ] + + expect(referential.stop_area_referential.stop_areas.pluck(*defined_attributes)).to eq(expected_attributes) + end + end + + describe "#import_routes" do + let(:import) { create_import "google-sample-feed.zip" } + it "should create a line for each route" do + import.import_routes + + defined_attributes = [ + :registration_number, :name, :number, :published_name, + "companies.registration_number", + :comment, :url + ] + expected_attributes = [ + ["AAMV", "Airport - Amargosa Valley", "50", "Airport - Amargosa Valley", nil, nil, nil], + ["CITY", "City", "40", "City", nil, nil, nil], + ["STBA", "Stagecoach - Airport Shuttle", "30", "Stagecoach - Airport Shuttle", nil, nil, nil], + ["BFC", "Bullfrog - Furnace Creek Resort", "20", "Bullfrog - Furnace Creek Resort", nil, nil, nil], + ["AB", "Airport - Bullfrog", "10", "Airport - Bullfrog", nil, nil, nil] + ] + + expect(referential.line_referential.lines.includes(:company).pluck(*defined_attributes)).to eq(expected_attributes) + end + end + + describe "#import_trips" do + let(:import) { create_import "google-sample-feed.zip" } + it "should create a Route for each trip" do + referential.switch + + import.import_routes + import.import_trips + + + defined_attributes = [ + "lines.registration_number", :wayback, :name, :published_name + ] + expected_attributes = [ + ["AB", "outbound", "to Bullfrog", "to Bullfrog"], + ["AB", "inbound", "to Airport", "to Airport"], + ["STBA", "inbound", "Shuttle", "Shuttle"], + ["CITY", "outbound", "Outbound", "Outbound"], + ["CITY", "inbound", "Inbound", "Inbound"], + ["BFC", "outbound", "to Furnace Creek Resort", "to Furnace Creek Resort"], + ["BFC", "inbound", "to Bullfrog", "to Bullfrog"], + ["AAMV", "outbound", "to Amargosa Valley", "to Amargosa Valley"], + ["AAMV", "inbound", "to Airport", "to Airport"], + ["AAMV", "outbound", "to Amargosa Valley", "to Amargosa Valley"], + ["AAMV", "inbound", "to Airport", "to Airport"] + ] + + expect(referential.routes.includes(:line).pluck(*defined_attributes)).to eq(expected_attributes) + end + + it "should create a JourneyPattern for each trip" do + referential.switch + + import.import_routes + import.import_trips + + defined_attributes = [ + :name + ] + expected_attributes = [ + "to Bullfrog", "to Airport", "Shuttle", "Outbound", "Inbound", "to Furnace Creek Resort", "to Bullfrog", "to Amargosa Valley", "to Airport", "to Amargosa Valley", "to Airport" + ] + + expect(referential.journey_patterns.pluck(*defined_attributes)).to eq(expected_attributes) + end + + it "should create a VehicleJourney for each trip" do + referential.switch + + import.import_routes + import.import_trips + + defined_attributes = [ + :published_journey_name + ] + expected_attributes = [ + "to Bullfrog", "to Airport", "Shuttle", "CITY1", "CITY2", "to Furnace Creek Resort", "to Bullfrog", "to Amargosa Valley", "to Airport", "to Amargosa Valley", "to Airport" + ] + + expect(referential.vehicle_journeys.pluck(*defined_attributes)).to eq(expected_attributes) + end + end + + describe "#import_stop_times" do + let(:import) { create_import "google-sample-feed.zip" } + + it "should create a VehicleJourneyAtStop for each stop_time" do + referential.switch + + import.import_stops + import.import_routes + import.import_trips + import.import_stop_times + + def t(value) + Time.parse(value) + end + + defined_attributes = [ + "stop_areas.registration_number", :position, :departure_time, :arrival_time, + ] + expected_attributes = [ + ["STAGECOACH", 0, t("2000-01-01 06:00:00 UTC"), t("2000-01-01 06:00:00 UTC")], + ["BEATTY_AIRPORT", 1, t("2000-01-01 06:20:00 UTC"), t("2000-01-01 06:20:00 UTC")], + ["STAGECOACH", 0, t("2000-01-01 06:00:00 UTC"), t("2000-01-01 06:00:00 UTC")], + ["NANAA", 1, t("2000-01-01 06:07:00 UTC"), t("2000-01-01 06:05:00 UTC")], + ["NADAV", 2, t("2000-01-01 06:14:00 UTC"), t("2000-01-01 06:12:00 UTC")], + ["DADAN", 3, t("2000-01-01 06:21:00 UTC"), t("2000-01-01 06:19:00 UTC")], + ["EMSI", 4, t("2000-01-01 06:28:00 UTC"), t("2000-01-01 06:26:00 UTC")], + ["EMSI", 0, t("2000-01-01 06:30:00 UTC"), t("2000-01-01 06:28:00 UTC")], + ["DADAN", 1, t("2000-01-01 06:37:00 UTC"), t("2000-01-01 06:35:00 UTC")], + ["NADAV", 2, t("2000-01-01 06:44:00 UTC"), t("2000-01-01 06:42:00 UTC")], + ["NANAA", 3, t("2000-01-01 06:51:00 UTC"), t("2000-01-01 06:49:00 UTC")], + ["STAGECOACH", 4, t("2000-01-01 06:58:00 UTC"), t("2000-01-01 06:56:00 UTC")], + ["BEATTY_AIRPORT", 0, t("2000-01-01 08:00:00 UTC"), t("2000-01-01 08:00:00 UTC")], + ["BULLFROG", 1, t("2000-01-01 08:15:00 UTC"), t("2000-01-01 08:10:00 UTC")], + ["BULLFROG", 0, t("2000-01-01 12:05:00 UTC"), t("2000-01-01 12:05:00 UTC")], + ["BEATTY_AIRPORT", 1, t("2000-01-01 12:15:00 UTC"), t("2000-01-01 12:15:00 UTC")], + ["BULLFROG", 0, t("2000-01-01 08:20:00 UTC"), t("2000-01-01 08:20:00 UTC")], + ["FUR_CREEK_RES", 1, t("2000-01-01 09:20:00 UTC"), t("2000-01-01 09:20:00 UTC")], + ["FUR_CREEK_RES", 0, t("2000-01-01 11:00:00 UTC"), t("2000-01-01 11:00:00 UTC")], + ["BULLFROG", 1, t("2000-01-01 12:00:00 UTC"), t("2000-01-01 12:00:00 UTC")], + ["BEATTY_AIRPORT", 0, t("2000-01-01 08:00:00 UTC"), t("2000-01-01 08:00:00 UTC")], + ["AMV", 1, t("2000-01-01 09:00:00 UTC"), t("2000-01-01 09:00:00 UTC")], + ["AMV", 0, t("2000-01-01 10:00:00 UTC"), t("2000-01-01 10:00:00 UTC")], + ["BEATTY_AIRPORT", 1, t("2000-01-01 11:00:00 UTC"), t("2000-01-01 11:00:00 UTC")], + ["BEATTY_AIRPORT", 0, t("2000-01-01 13:00:00 UTC"), t("2000-01-01 13:00:00 UTC")], + ["AMV", 1, t("2000-01-01 14:00:00 UTC"), t("2000-01-01 14:00:00 UTC")], + ["AMV", 0, t("2000-01-01 15:00:00 UTC"), t("2000-01-01 15:00:00 UTC")], + ["BEATTY_AIRPORT", 1, t("2000-01-01 16:00:00 UTC"), t("2000-01-01 16:00:00 UTC")] + ] + expect(referential.vehicle_journey_at_stops.includes(stop_point: :stop_area).pluck(*defined_attributes)).to eq(expected_attributes) + end + end + +end -- cgit v1.2.3