diff --git a/.gitignore b/.gitignore index 59c74047..2eec9120 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /tmp /log /public +/.idea \ No newline at end of file diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..c99d2e73 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 00000000..89c7595f --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,182 @@ +plugins: + - rubocop-minitest + - rubocop-performance + - rubocop-rspec + - rubocop-rails + +AllCops: + SuggestExtensions: false + NewCops: enable + TargetRubyVersion: 3.4 + Exclude: + - 'vendor/**/*' + - 'db/schema.rb' + - 'db/migrate/*' + - 'bin/*' + - 'node_modules/**/*' + - 'tmp/**/*' + - 'log/**/*' + +# ОСНОВНЫЕ СТИЛЕВЫЕ ПРАВИЛА + +# Предпочитаем ДВОЙНЫЕ кавычки +Style/StringLiterals: + Enabled: true + EnforcedStyle: double_quotes + +Style/StringLiteralsInInterpolation: + Enabled: true + EnforcedStyle: double_quotes + +# Отключаем требование документации +Style/Documentation: + Enabled: false + +# Разрешаем не-ASCII комментарии (русские комментарии) +Style/AsciiComments: + Enabled: false + +# МЕТРИКИ И ДЛИНЫ + +Layout/LineLength: + Max: 200 + AllowURI: true + AllowHeredoc: true + +Metrics/MethodLength: + Max: 100 + CountAsOne: ['array', 'hash', 'heredoc'] + +Metrics/ClassLength: + Max: 200 + CountAsOne: ['array', 'hash', 'heredoc'] + +Metrics/ModuleLength: + Max: 200 + +Metrics/BlockLength: + Max: 100 + CountAsOne: ['array', 'hash', 'heredoc'] + Exclude: + - 'spec/**/*' + - 'config/routes.rb' + +Metrics/AbcSize: + Max: 20 + +Metrics/CyclomaticComplexity: + Max: 15 + +Metrics/PerceivedComplexity: + Max: 15 + +# LAYOUT И ФОРМАТИРОВАНИЕ + +Layout/EmptyLinesAroundAttributeAccessor: + Enabled: true + +Layout/SpaceAroundMethodCallOperator: + Enabled: true + +Layout/MultilineMethodCallIndentation: + Enabled: true + EnforcedStyle: indented + +# RAILS-СПЕЦИФИЧНЫЕ ПРАВИЛА + +Rails/FilePath: + Enabled: false + +Rails/UnknownEnv: + Environments: + - production + - development + - test + +Rails/SkipsModelValidations: + Enabled: false # Для bulk import операций + +# RSPEC-СПЕЦИФИЧНЫЕ ПРАВИЛА + +RSpec/ExampleLength: + Max: 20 + +RSpec/MultipleExpectations: + Max: 5 + +RSpec/NestedGroups: + Max: 4 + +RSpec/DescribeClass: + Enabled: false + +# PERFORMANCE ПРАВИЛА + +Performance/StringReplacement: + Enabled: true + +Performance/RedundantBlockCall: + Enabled: true + +Performance/StringInclude: + Enabled: true + +# БЕЗОПАСНОСТЬ + +Security/Open: + Enabled: true + +Security/YAMLLoad: + Enabled: true + +# ИМЕНОВАНИЕ + +Naming/PredicatePrefix: + Enabled: true + AllowedMethods: + - is_a? + - has_key? + +Naming/MemoizedInstanceVariableName: + Enabled: false + +# STYLE ПРАВИЛА + +Style/HashEachMethods: + Enabled: true + +Style/HashTransformKeys: + Enabled: true + +Style/HashTransformValues: + Enabled: true + +Style/RedundantRegexpEscape: + Enabled: false # Может конфликтовать с некоторыми регексами + +Style/FrozenStringLiteralComment: + Enabled: true + EnforcedStyle: always + +Style/SymbolArray: + Enabled: true + EnforcedStyle: brackets + +Style/WordArray: + Enabled: true + EnforcedStyle: brackets + Exclude: + - 'app/models/**/*' + +# Разрешаем trailing запятые +Style/TrailingCommaInArguments: + Enabled: true + EnforcedStyleForMultiline: comma + +Style/TrailingCommaInArrayLiteral: + Enabled: true + EnforcedStyleForMultiline: comma + +Style/TrailingCommaInHashLiteral: + Enabled: true + EnforcedStyleForMultiline: comma diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml new file mode 100644 index 00000000..7f609f6d --- /dev/null +++ b/.rubocop_todo.yml @@ -0,0 +1,7 @@ +# This configuration was generated by +# `rubocop --auto-gen-config` +# and is meant to be a temporary file to ease migration to stricter rules. +# Remove or comment out rules as you fix the violations. + +# Uncomment below if you have many violations and want to fix them gradually +# inherit_from: .rubocop_todo.yml diff --git a/.ruby-gemset b/.ruby-gemset new file mode 100644 index 00000000..823585cd --- /dev/null +++ b/.ruby-gemset @@ -0,0 +1 @@ +optimization \ No newline at end of file diff --git a/.ruby-version b/.ruby-version index 47b322c9..f9892605 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.4.1 +3.4.4 diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..3e1ce0e7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,25 @@ +{ + "ruby.useLanguageServer": true, + "ruby.format": "rubocop", + "ruby.lint": { + "rubocop": true, + "ruby": false + }, + "ruby.rubocop.executePath": "bundle exec rubocop", + "ruby.rubocop.configFilePath": ".rubocop.yml", + "editor.formatOnSave": true, + "editor.formatOnPaste": true, + "files.trimTrailingWhitespace": true, + "files.insertFinalNewline": true, + "files.trimFinalNewlines": true, + "[ruby]": { + "editor.defaultFormatter": "rebornix.Ruby", + "editor.formatOnSave": true, + "editor.rulers": [200], + "editor.tabSize": 2, + "editor.insertSpaces": true + }, + "emmet.includeLanguages": { + "erb": "html" + } +} diff --git a/Gemfile b/Gemfile index 34074dfd..35f7f110 100644 --- a/Gemfile +++ b/Gemfile @@ -1,14 +1,54 @@ -source 'https://rubygems.org' +# frozen_string_literal: true + +source "https://rubygems.org" git_source(:github) { |repo| "https://github.com/#{repo}.git" } -ruby file: '.ruby-version' +ruby file: ".ruby-version" + +# Core Rails gems +gem "bootsnap" +gem "listen" +gem "pg" +gem "puma" +gem "rails", "~> 8.0.1" +gem "sprockets-rails" -gem 'rails', '~> 8.0.1' -gem 'pg' -gem 'puma' -gem 'listen' -gem 'bootsnap' -gem 'rack-mini-profiler' +# Core application dependencies +gem "activerecord-import" +gem "oj" +gem "ruby-progressbar" # Windows does not include zoneinfo files, so bundle the tzinfo-data gem -gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] +gem "tzinfo-data", platforms: [:mingw, :mswin, :x64_mingw, :jruby] + +group :development do + # Database monitoring and optimization + gem "bullet" + gem "pghero" + gem "pg_query", ">= 2" + gem "strong_migrations" +end + +group :development, :test do + # Code quality and linting + gem "rubocop", require: false + gem "rubocop-minitest", require: false + gem "rubocop-performance", require: false + gem "rubocop-rails", require: false + gem "rubocop-rspec", require: false + + # Profiling tools + gem "memory_profiler", require: false + gem "pry" + gem "rack-mini-profiler", require: false + gem "ruby-prof", require: false + gem "stackprof", require: false + gem "test-prof", "~> 1.4.4" + gem "vernier", "~> 1.7", require: false +end + +group :test do + gem "profile-viewer", require: false + gem "rspec-rails" + gem "rspec-sqlimit" +end diff --git a/Gemfile.lock b/Gemfile.lock index a9ddd818..17a97d29 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,6 +53,8 @@ GEM activemodel (= 8.0.1) activesupport (= 8.0.1) timeout (>= 0.4.0) + activerecord-import (2.2.0) + activerecord (>= 4.2) activestorage (8.0.1) actionpack (= 8.0.1) activejob (= 8.0.1) @@ -72,21 +74,30 @@ GEM securerandom (>= 0.3) tzinfo (~> 2.0, >= 2.0.5) uri (>= 0.13.1) + ast (2.4.3) base64 (0.2.0) benchmark (0.4.0) bigdecimal (3.1.9) bootsnap (1.18.4) msgpack (~> 1.2) builder (3.3.0) + bullet (8.0.8) + activesupport (>= 3.0.0) + uniform_notifier (~> 1.11) + coderay (1.1.3) concurrent-ruby (1.3.5) connection_pool (2.5.0) crass (1.0.6) date (3.4.1) + diff-lcs (1.6.2) drb (2.2.1) erubi (1.13.1) - ffi (1.17.1-arm64-darwin) + ffi (1.17.2-arm64-darwin) globalid (1.2.1) activesupport (>= 6.1) + google-protobuf (4.32.0-arm64-darwin) + bigdecimal + rake (>= 13) i18n (1.14.7) concurrent-ruby (~> 1.0) io-console (0.8.0) @@ -94,6 +105,9 @@ GEM pp (>= 0.6.0) rdoc (>= 4.0.0) reline (>= 0.4.2) + json (2.13.2) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) @@ -107,6 +121,8 @@ GEM net-pop net-smtp marcel (1.0.4) + memory_profiler (1.1.0) + method_source (1.1.0) mini_mime (1.1.5) minitest (5.25.4) msgpack (1.8.0) @@ -122,10 +138,30 @@ GEM nio4r (2.7.4) nokogiri (1.18.2-arm64-darwin) racc (~> 1.4) + oj (3.16.11) + bigdecimal (>= 3.0) + ostruct (>= 0.2) + optparse (0.6.0) + ostruct (0.6.3) + parallel (1.27.0) + parser (3.3.9.0) + ast (~> 2.4.1) + racc pg (1.5.9) + pg_query (6.1.0) + google-protobuf (>= 3.25.3) + pghero (3.7.0) + activerecord (>= 7.1) pp (0.6.2) prettyprint prettyprint (0.2.0) + prism (1.4.0) + profile-viewer (0.0.5) + optparse + webrick + pry (0.15.2) + coderay (~> 1.1) + method_source (~> 1.0) psych (5.2.3) date stringio @@ -133,7 +169,7 @@ GEM nio4r (~> 2.0) racc (1.8.1) rack (3.1.9) - rack-mini-profiler (3.3.1) + rack-mini-profiler (4.0.1) rack (>= 1.2.0) rack-session (2.1.0) base64 (>= 0.1.0) @@ -171,22 +207,100 @@ GEM rake (>= 12.2) thor (~> 1.0, >= 1.2.2) zeitwerk (~> 2.6) + rainbow (3.1.1) rake (13.2.1) rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) rdoc (6.12.0) psych (>= 4.0.0) + regexp_parser (2.11.2) reline (0.6.0) io-console (~> 0.5) + rspec (3.13.1) + rspec-core (~> 3.13.0) + rspec-expectations (~> 3.13.0) + rspec-mocks (~> 3.13.0) + rspec-core (3.13.5) + rspec-support (~> 3.13.0) + rspec-expectations (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-mocks (3.13.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.13.0) + rspec-rails (8.0.2) + actionpack (>= 7.2) + activesupport (>= 7.2) + railties (>= 7.2) + rspec-core (~> 3.13) + rspec-expectations (~> 3.13) + rspec-mocks (~> 3.13) + rspec-support (~> 3.13) + rspec-sqlimit (1.0.0) + activerecord (>= 4.2.0) + rspec (~> 3.0) + rspec-support (3.13.4) + rubocop (1.79.2) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.46.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.46.0) + parser (>= 3.3.7.2) + prism (~> 1.4) + rubocop-minitest (0.38.1) + lint_roller (~> 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.38.0, < 2.0) + rubocop-performance (1.25.0) + lint_roller (~> 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.38.0, < 2.0) + rubocop-rails (2.33.3) + activesupport (>= 4.2.0) + lint_roller (~> 1.1) + rack (>= 1.1) + rubocop (>= 1.75.0, < 2.0) + rubocop-ast (>= 1.44.0, < 2.0) + rubocop-rspec (3.6.0) + lint_roller (~> 1.1) + rubocop (~> 1.72, >= 1.72.1) + ruby-prof (1.7.2) + base64 + ruby-progressbar (1.13.0) securerandom (0.4.1) + sprockets (4.2.2) + concurrent-ruby (~> 1.0) + logger + rack (>= 2.2.4, < 4) + sprockets-rails (3.5.2) + actionpack (>= 6.1) + activesupport (>= 6.1) + sprockets (>= 3.0.0) + stackprof (0.2.27) stringio (3.1.2) + strong_migrations (2.5.0) + activerecord (>= 7.1) + test-prof (1.4.4) thor (1.3.2) timeout (0.4.3) tzinfo (2.0.6) concurrent-ruby (~> 1.0) + unicode-display_width (3.1.5) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) + uniform_notifier (1.17.0) uri (1.0.2) useragent (0.16.11) + vernier (1.8.0) + webrick (1.9.1) websocket-driver (0.7.7) base64 websocket-extensions (>= 0.1.0) @@ -197,16 +311,38 @@ PLATFORMS arm64-darwin-24 DEPENDENCIES + activerecord-import bootsnap + bullet listen + memory_profiler + oj pg + pg_query (>= 2) + pghero + profile-viewer + pry puma rack-mini-profiler rails (~> 8.0.1) + rspec-rails + rspec-sqlimit + rubocop + rubocop-minitest + rubocop-performance + rubocop-rails + rubocop-rspec + ruby-prof + ruby-progressbar + sprockets-rails + stackprof + strong_migrations + test-prof (~> 1.4.4) tzinfo-data + vernier (~> 1.7) RUBY VERSION - ruby 3.4.1p0 + ruby 3.4.4p34 BUNDLED WITH 2.6.2 diff --git a/Rakefile b/Rakefile index e85f9139..d2a78aa2 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,8 @@ +# frozen_string_literal: true + # Add your own tasks in files placed in lib/tasks ending in .rake, # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. -require_relative 'config/application' +require_relative "config/application" Rails.application.load_tasks diff --git a/app/channels/application_cable/channel.rb b/app/channels/application_cable/channel.rb index d6726972..9aec2305 100644 --- a/app/channels/application_cable/channel.rb +++ b/app/channels/application_cable/channel.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ApplicationCable class Channel < ActionCable::Channel::Base end diff --git a/app/channels/application_cable/connection.rb b/app/channels/application_cable/connection.rb index 0ff5442f..8d6c2a1b 100644 --- a/app/channels/application_cable/connection.rb +++ b/app/channels/application_cable/connection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ApplicationCable class Connection < ActionCable::Connection::Base end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 09705d12..7944f9f9 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class ApplicationController < ActionController::Base end diff --git a/app/controllers/trips_controller.rb b/app/controllers/trips_controller.rb index acb38be2..1490d713 100644 --- a/app/controllers/trips_controller.rb +++ b/app/controllers/trips_controller.rb @@ -1,7 +1,9 @@ +# frozen_string_literal: true + class TripsController < ApplicationController def index - @from = City.find_by_name!(params[:from]) - @to = City.find_by_name!(params[:to]) + @from = City.find_by!(name: params[:from]) + @to = City.find_by!(name: params[:to]) @trips = Trip.where(from: @from, to: @to).order(:start_time) end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index de6be794..15b06f0f 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + module ApplicationHelper end diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index a009ace5..d92ffddc 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,2 +1,4 @@ +# frozen_string_literal: true + class ApplicationJob < ActiveJob::Base end diff --git a/app/mailers/application_mailer.rb b/app/mailers/application_mailer.rb index 286b2239..5cc63a0c 100644 --- a/app/mailers/application_mailer.rb +++ b/app/mailers/application_mailer.rb @@ -1,4 +1,6 @@ +# frozen_string_literal: true + class ApplicationMailer < ActionMailer::Base - default from: 'from@example.com' - layout 'mailer' + default from: "from@example.com" + layout "mailer" end diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 10a4cba8..71fbba5b 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + class ApplicationRecord < ActiveRecord::Base self.abstract_class = true end diff --git a/app/models/bus.rb b/app/models/bus.rb index 1dcc54cb..7b96d9b9 100644 --- a/app/models/bus.rb +++ b/app/models/bus.rb @@ -1,16 +1,7 @@ +# frozen_string_literal: true + class Bus < ApplicationRecord - MODELS = [ - 'Икарус', - 'Мерседес', - 'Сканиа', - 'Буханка', - 'УАЗ', - 'Спринтер', - 'ГАЗ', - 'ПАЗ', - 'Вольво', - 'Газель', - ].freeze + MODELS = %w[Икарус Мерседес Сканиа Буханка УАЗ Спринтер ГАЗ ПАЗ Вольво Газель].freeze has_many :trips has_and_belongs_to_many :services, join_table: :buses_services diff --git a/app/models/city.rb b/app/models/city.rb index 19ec7f36..fd2ef235 100644 --- a/app/models/city.rb +++ b/app/models/city.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + class City < ApplicationRecord validates :name, presence: true, uniqueness: true validate :name_has_no_spaces def name_has_no_spaces - errors.add(:name, "has spaces") if name.include?(' ') + errors.add(:name, "has spaces") if name.include?(" ") end end diff --git a/app/models/service.rb b/app/models/service.rb index 9cbb2a32..707b4993 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -1,15 +1,17 @@ +# frozen_string_literal: true + class Service < ApplicationRecord SERVICES = [ - 'WiFi', - 'Туалет', - 'Работающий туалет', - 'Ремни безопасности', - 'Кондиционер общий', - 'Кондиционер Индивидуальный', - 'Телевизор общий', - 'Телевизор индивидуальный', - 'Стюардесса', - 'Можно не печатать билет', + "WiFi", + "Туалет", + "Работающий туалет", + "Ремни безопасности", + "Кондиционер общий", + "Кондиционер Индивидуальный", + "Телевизор общий", + "Телевизор индивидуальный", + "Стюардесса", + "Можно не печатать билет", ].freeze has_and_belongs_to_many :buses, join_table: :buses_services diff --git a/app/models/trip.rb b/app/models/trip.rb index 9d63dfff..d9369f9b 100644 --- a/app/models/trip.rb +++ b/app/models/trip.rb @@ -1,15 +1,13 @@ +# frozen_string_literal: true + class Trip < ApplicationRecord HHMM_REGEXP = /([0-1][0-9]|[2][0-3]):[0-5][0-9]/ - belongs_to :from, class_name: 'City' - belongs_to :to, class_name: 'City' + belongs_to :from, class_name: "City" + belongs_to :to, class_name: "City" belongs_to :bus - validates :from, presence: true - validates :to, presence: true - validates :bus, presence: true - - validates :start_time, format: { with: HHMM_REGEXP, message: 'Invalid time' } + validates :start_time, format: { with: HHMM_REGEXP, message: "Invalid time" } validates :duration_minutes, presence: true validates :duration_minutes, numericality: { greater_than: 0 } validates :price_cents, presence: true diff --git a/app/services/json_trip_loader.rb b/app/services/json_trip_loader.rb new file mode 100644 index 00000000..6ce55453 --- /dev/null +++ b/app/services/json_trip_loader.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require "benchmark" +require "ruby-progressbar" + +class JsonTripLoader + def load!(path) + json = JSON.parse(File.read(path)) + + old_logger = ActiveRecord::Base.logger + old_verbose = + ActiveRecord::Base.respond_to?(:verbose_query_logs) ? ActiveRecord::Base.verbose_query_logs : nil + + time = Benchmark.realtime do + # тихий режим + ActiveRecord::Base.logger = nil + ActiveRecord::Base.verbose_query_logs = false if ActiveRecord::Base.respond_to?(:verbose_query_logs=) + + ActiveRecord::Base.transaction do + ActiveRecord::Base.connection.execute <<~SQL.squish + TRUNCATE TABLE + buses_services, + trips, + buses, + services, + cities + RESTART IDENTITY + CASCADE; + SQL + + city_id_cache = {} + service_id_cache = {} + bus_cache = {} + trips = [] + + progress_bar = ProgressBar.create( + title: "Loading trips", + total: json.size, + format: "%t: |%B| %p%% %e", + ) + + json.each do |trip| + from_name = trip["from"] + to_name = trip["to"] + + from_id = city_id_cache[from_name] ||= City.find_or_create_by(name: from_name).id + to_id = city_id_cache[to_name] ||= City.find_or_create_by(name: to_name).id + + service_ids = trip["bus"]["services"].map do |name| + service_id_cache[name] ||= Service.find_or_create_by(name:).id + end + + number = trip["bus"]["number"] + model = trip["bus"]["model"] + bus = bus_cache[number] ||= Bus.find_or_create_by(number:) + bus.model = model if bus.model != model + bus.service_ids = service_ids if bus.service_ids != service_ids + bus.save if bus.changed? + + # Используем массив - самый быстрый вариант + trips << [ + from_id, + to_id, + bus.id, + trip["start_time"], + trip["duration_minutes"], + trip["price_cents"], + ] + + progress_bar.increment + end + + # Указываем колонки явно при импорте массивов + Trip.import( + [:from_id, :to_id, :bus_id, :start_time, :duration_minutes, :price_cents], + trips, + validate: false, + ) + end + ensure + ActiveRecord::Base.logger = old_logger + ActiveRecord::Base.verbose_query_logs = old_verbose if ActiveRecord::Base.respond_to?(:verbose_query_logs=) + end + + Rails.logger.debug { "JsonTripLoader.load!: #{time.round(3)}s (#{json.size} trips)" } + end +end diff --git a/app/services/json_trip_loader_ai.rb b/app/services/json_trip_loader_ai.rb new file mode 100644 index 00000000..43811240 --- /dev/null +++ b/app/services/json_trip_loader_ai.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +require "benchmark" +require "ruby-progressbar" + +class JsonTripLoaderAi + BATCH_SIZE = 5_000 + + # Если нет модели для join-таблицы — объявим локально. + unless defined?(BusService) + class BusService < ApplicationRecord + self.table_name = "buses_services" + end + end + + def load!(path) + json = JSON.parse(File.read(path)) + + old_logger = ActiveRecord::Base.logger + old_verbose = + ActiveRecord::Base.respond_to?(:verbose_query_logs) ? ActiveRecord::Base.verbose_query_logs : nil + + time = Benchmark.realtime do + # Тихий режим логгера, чтобы не тратить время на форматирование логов + ActiveRecord::Base.logger = nil + ActiveRecord::Base.verbose_query_logs = false if ActiveRecord::Base.respond_to?(:verbose_query_logs=) + + ActiveRecord::Base.transaction do + # Быстрый сброс + ActiveRecord::Base.connection.execute <<~SQL.squish + TRUNCATE TABLE + buses_services, + trips, + buses, + services, + cities + RESTART IDENTITY + CASCADE; + SQL + + # 1) Собираем уникальные сущности из JSON + city_names = Set.new + service_names = Set.new + bus_number_to_model = {} # number => model + bus_number_to_service_names = Hash.new { |h, k| h[k] = Set.new } + + json.each do |trip| + from_name = trip["from"] + to_name = trip["to"] + city_names << from_name + city_names << to_name + + number = trip["bus"]["number"] + model = trip["bus"]["model"] + bus_number_to_model[number] = model + + (trip["bus"]["services"] || []).each do |sname| + service_names << sname + bus_number_to_service_names[number] << sname + end + end + + # 2) Вставляем города/сервисы одним махом (ignore duplicates) + if city_names.any? + City.import( + city_names.map { |n| City.new(name: n) }, + on_duplicate_key_ignore: true, + validate: false, + ) + end + + if service_names.any? + Service.import( + service_names.map { |n| Service.new(name: n) }, + on_duplicate_key_ignore: true, + validate: false, + ) + end + + # 3) Получаем маппинги name -> id одним запросом на каждую таблицу + city_id_by_name = City.where(name: city_names.to_a).pluck(:name, :id).to_h + service_id_by_name = Service.where(name: service_names.to_a).pluck(:name, :id).to_h + + # 4) Upsert автобусов (обновляем model при конфликте по number) + if bus_number_to_model.any? + # Через activerecord-import с on_duplicate_key_update + Bus.import( + bus_number_to_model.map { |number, model| Bus.new(number: number, model: model) }, + on_duplicate_key_update: [:model], + validate: false, + ) + end + + # 5) Получаем маппинг автобусов number -> id одним запросом + bus_id_by_number = Bus.where(number: bus_number_to_model.keys).pluck(:number, :id).to_h + + # 6) Пакетно вставляем связи bus↔service в join-таблицу (ignore duplicates) + join_rows = [] + bus_number_to_service_names.each do |number, sset| + bus_id = bus_id_by_number[number] + sset.each do |sname| + sid = service_id_by_name[sname] + next unless bus_id && sid + + join_rows << { bus_id: bus_id, service_id: sid } + end + end + + if join_rows.any? + # Через модель BusService — activerecord-import игнорирует дубликаты + BusService.import( + join_rows, + on_duplicate_key_ignore: true, + validate: false, + ) + end + + # 7) Формируем и заливаем trips батчами + trips_buffer = [] + trips_buffer.reserve([json.size, BATCH_SIZE].min) if trips_buffer.respond_to?(:reserve) + + flush_trips = lambda do + next if trips_buffer.empty? + + Trip.import(trips_buffer, validate: true, batch_size: BATCH_SIZE) + trips_buffer.clear + end + + # прогрессбар на количество записей + progress_bar = ProgressBar.create( + title: "Loading trips", + total: json.size, + format: "%t: |%B| %p%% %e", + ) + + json.each do |t| + trips_buffer << Trip.new( + from_id: city_id_by_name[t["from"]], + to_id: city_id_by_name[t["to"]], + bus_id: bus_id_by_number[t["bus"]["number"]], + start_time: t["start_time"], + duration_minutes: t["duration_minutes"], + price_cents: t["price_cents"], + ) + flush_trips.call if trips_buffer.size >= BATCH_SIZE + + progress_bar.increment + end + flush_trips.call + end + ensure + ActiveRecord::Base.logger = old_logger + ActiveRecord::Base.verbose_query_logs = old_verbose if ActiveRecord::Base.respond_to?(:verbose_query_logs=) + end + + Rails.logger.debug { "JsonTripLoader.load!: #{time.round(3)}s (#{json.size} trips)" } + end +end diff --git a/app/services/json_trip_loader_stream.rb b/app/services/json_trip_loader_stream.rb new file mode 100644 index 00000000..f652abe2 --- /dev/null +++ b/app/services/json_trip_loader_stream.rb @@ -0,0 +1,178 @@ +# frozen_string_literal: true + +require "benchmark" +require "ruby-progressbar" +require "oj" # Быстрый JSON парсер + +class JsonTripLoaderStream + BATCH_SIZE = 5000 + + def load!(path) + old_logger = ActiveRecord::Base.logger + old_verbose = ActiveRecord::Base.respond_to?(:verbose_query_logs) ? ActiveRecord::Base.verbose_query_logs : nil + + time = Benchmark.realtime do + ActiveRecord::Base.logger = nil + ActiveRecord::Base.verbose_query_logs = false if ActiveRecord::Base.respond_to?(:verbose_query_logs=) + + ActiveRecord::Base.transaction do + clear_database + + # Кэши для уникальных значений + @city_cache = {} + @service_cache = {} + @bus_cache = {} + @trips_batch = [] + + # Получаем размер файла для прогресс-бара (приблизительно) + file_size = File.size(path) + progress_bar = ProgressBar.create( + title: "Loading trips", + total: file_size, + format: "%t: |%B| %p%% %e", + ) + + # Потоковое чтение и парсинг + stream_parse_file(path, progress_bar) + + # Импортируем последний батч + import_trips_batch if @trips_batch.any? + end + ensure + ActiveRecord::Base.logger = old_logger + ActiveRecord::Base.verbose_query_logs = old_verbose if ActiveRecord::Base.respond_to?(:verbose_query_logs=) + end + + Rails.logger.debug { "JsonTripLoader.load!: #{time.round(3)}s" } + end + + private + + def clear_database + ActiveRecord::Base.connection.execute <<~SQL.squish + TRUNCATE TABLE + buses_services, + trips, + buses, + services, + cities + RESTART IDENTITY + CASCADE; + SQL + end + + def stream_parse_file(path, progress_bar) + File.open(path, "r") do |file| + nesting = 0 + str = +"" + bytes_read = 0 + in_array = false + + until file.eof? + ch = file.read(1) + bytes_read += 1 + + # Обновляем прогресс каждые 10KB + progress_bar.progress = bytes_read if (bytes_read % 10_000).zero? + + case ch + when "[" + in_array = true if nesting.zero? + str << ch if nesting.positive? + when "]" + # Конец корневого массива + if nesting.zero? + in_array = false + next + end + str << ch + when "{" + nesting += 1 + str << ch + when "}" + str << ch + nesting -= 1 + + # Если закончился объект верхнего уровня (trip) + if nesting.zero? && in_array + begin + trip = Oj.load(str) + process_trip(trip) + rescue Oj::ParseError => e + Rails.logger.debug { "Parse error: #{e.message}" } + Rails.logger.debug { "String: #{str[0..100]}..." } if str.length > 100 + raise + end + str = +"" + end + when "," + # Запятая между объектами - игнорируем если на верхнем уровне + next if nesting.zero? + + str << ch + when /\s/ + # Пробелы - игнорируем если на верхнем уровне + next if nesting.zero? + + str << ch + else + str << ch if nesting.positive? + end + end + + progress_bar.finish + end + end + + def process_trip(trip) + from_id = get_or_create_city_id(trip["from"]) + to_id = get_or_create_city_id(trip["to"]) + bus_id = get_or_create_bus_id(trip["bus"]) + + @trips_batch << [ + from_id, + to_id, + bus_id, + trip["start_time"], + trip["duration_minutes"], + trip["price_cents"], + ] + + # Импортируем батч когда набрался нужный размер + return unless @trips_batch.size >= BATCH_SIZE + + import_trips_batch + @trips_batch.clear + end + + def get_or_create_city_id(name) + @city_cache[name] ||= City.find_or_create_by(name: name).id + end + + def get_or_create_bus_id(bus_data) + number = bus_data["number"] + + @bus_cache[number] ||= begin + bus = Bus.find_or_create_by(number: number) + bus.model = bus_data["model"] if bus.model != bus_data["model"] + + service_ids = bus_data["services"].map { |name| get_or_create_service_id(name) } + bus.service_ids = service_ids if bus.service_ids != service_ids + + bus.save if bus.changed? + bus.id + end + end + + def get_or_create_service_id(name) + @service_cache[name] ||= Service.find_or_create_by(name: name).id + end + + def import_trips_batch + Trip.import( + [:from_id, :to_id, :bus_id, :start_time, :duration_minutes, :price_cents], + @trips_batch, + validate: false, + ) + end +end diff --git a/app/views/trips/index.html.erb b/app/views/trips/index.html.erb index a60bce41..79bc535a 100644 --- a/app/views/trips/index.html.erb +++ b/app/views/trips/index.html.erb @@ -5,7 +5,7 @@ <%= "В расписании #{@trips.count} рейсов" %> -<% @trips.each do |trip| %> +<% @trips.preload([bus: [:services]]).each do |trip| %>