Merge pull request #1177 from Freika/feature/store-geodata

Feature/store geodata

Author: Freika

CHANGELOG.md

@@ -5,12 +5,38 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 
-# 0.26.1 - 2025-05-12
+# 0.26.1 - 2025-05-15
+
+## Geodata on demand
+
+This release introduces a new environment variable `STORE_GEODATA` to control whether to store geodata in the database.
+
+When `STORE_GEODATA` is disabled, each feature that uses geodata will now make a direct request to the geocoding service to calculate required data.
+
+Geodata is being used:
+
+- Fetching places geodata
+- Fetching countries for a trip
+- Suggesting place name for a visit
+
+If you prefer to keep the old behavior, you can set `STORE_GEODATA` to `true`. By default, starting this release, it's set to `false`.
+
+If you're running your own Photon instance, you can safely set `STORE_GEODATA` to `false`, otherwise it'd be better to keep it enabled, because that way Dawarich will be using existing geodata for its calculations.
+
+## Added
+
+- Map page now has a button to go to the previous and next day. #296 #631 #904
+
+## Changed
+
+- Reverse geocoding is now working as on-demand job instead of storing the result in the database.
+- Stats cards now show the last update time. #733
 
 ## Fixed
 
 - Fixed a bug with an attempt to write points with same lonlat and timestamp from iOS app. #1170
 - Importing GeoJSON files now saves velocity if it was stored in either `velocity` or `speed` property.
+- `rake points:migrate_to_lonlat` should work properly now. #1083 #1161
 
 
 # 0.26.0 - 2025-05-08

Gemfile

@@ -30,6 +30,7 @@ gem 'rails', '~> 8.0'
 gem 'rexml'
 gem 'rgeo'
 gem 'rgeo-activerecord'
+gem 'rgeo-geojson'
 gem 'rswag-api'
 gem 'rswag-ui'
 gem 'sentry-ruby'

Gemfile.lock

@@ -219,6 +219,7 @@ GEM
     mini_portile2 (2.8.8)
     minitest (5.25.5)
     msgpack (1.7.3)
+    multi_json (1.15.0)
     multi_xml (0.7.1)
       bigdecimal (~> 3.1)
     net-imap (0.5.8)
@@ -339,6 +340,9 @@ GEM
     rgeo-activerecord (8.0.0)
       activerecord (>= 7.0)
       rgeo (>= 3.0)
+    rgeo-geojson (2.2.0)
+      multi_json (~> 1.15)
+      rgeo (>= 1.0.0)
     rspec-core (3.13.3)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.3)
@@ -513,6 +517,7 @@ DEPENDENCIES
   rexml
   rgeo
   rgeo-activerecord
+  rgeo-geojson
   rspec-rails
   rswag-api
   rswag-specs

app/assets/builds/tailwind.css

@@ -5,7 +5,7 @@ class StatsController < ApplicationController
   before_action :authenticate_active_user!, only: %i[update update_all]
 
   def index
-    @stats = current_user.stats.group_by(&:year).sort.reverse
+    @stats = current_user.stats.group_by(&:year).transform_values { |stats| stats.sort_by(&:updated_at).reverse }.sort.reverse
     @points_total = current_user.tracked_points.count
     @points_reverse_geocoded = current_user.total_reverse_geocoded_points
     @points_reverse_geocoded_without_data = current_user.total_reverse_geocoded_points_without_data

app/controllers/stats_controller.rb

@@ -6,7 +6,7 @@ class Trips::CreatePathJob < ApplicationJob
   def perform(trip_id)
     trip = Trip.find(trip_id)
 
-    trip.calculate_path_and_distance
+    trip.calculate_trip_data
 
     trip.save!
   end

app/jobs/trips/create_path_job.rb

@@ -22,29 +22,19 @@ def lat
     lonlat.y
   end
 
-  def async_reverse_geocode
-    return unless DawarichSettings.reverse_geocoding_enabled?
-
-    ReverseGeocodingJob.perform_later(self.class.to_s, id)
-  end
-
-  def reverse_geocoded?
-    geodata.present?
-  end
-
   def osm_id
-    geodata['properties']['osm_id']
+    geodata.dig('properties', 'osm_id')
   end
 
   def osm_key
-    geodata['properties']['osm_key']
+    geodata.dig('properties', 'osm_key')
   end
 
   def osm_value
-    geodata['properties']['osm_value']
+    geodata.dig('properties', 'osm_value')
   end
 
   def osm_type
-    geodata['properties']['osm_type']
+    geodata.dig('properties', 'osm_type')
   end
 end

app/models/place.rb

@@ -28,7 +28,7 @@ class Point < ApplicationRecord
   scope :visited, -> { where.not(visit_id: nil) }
   scope :not_visited, -> { where(visit_id: nil) }
 
-  after_create :async_reverse_geocode
+  after_create :async_reverse_geocode, if: -> { DawarichSettings.store_geodata? }
   after_create_commit :broadcast_coordinates
 
   def self.without_raw_data

app/models/point.rb

@@ -7,19 +7,22 @@ class Trip < ApplicationRecord
 
   validates :name, :started_at, :ended_at, presence: true
 
-  before_save :calculate_path_and_distance
+  before_save :calculate_trip_data
 
-  def calculate_path_and_distance
+  def calculate_trip_data
     calculate_path
     calculate_distance
+    calculate_countries
   end
 
   def points
     user.tracked_points.where(timestamp: started_at.to_i..ended_at.to_i).order(:timestamp)
   end
 
   def countries
-    points.pluck(:country).uniq.compact
+    return points.pluck(:country).uniq.compact if DawarichSettings.store_geodata?
+
+    visited_countries
   end
 
   def photo_previews
@@ -56,4 +59,10 @@ def calculate_distance
 
     self.distance = distance.round
   end
+
+  def calculate_countries
+    countries = Trips::Countries.new(self).call
+
+    self.visited_countries = countries
+  end
 end

app/models/trip.rb

@@ -12,10 +12,6 @@ class Visit < ApplicationRecord
 
   enum :status, { suggested: 0, confirmed: 1, declined: 2 }
 
-  def reverse_geocoded?
-    place.geodata.present?
-  end
-
   def coordinates
     points.pluck(:latitude, :longitude).map { [_1[0].to_f, _1[1].to_f] }
   end

app/models/visit.rb

@@ -7,14 +7,16 @@ def initialize(place)
 
   def call
     {
-      id: place.id,
-      name: place.name,
-      longitude: place.lon,
-      latitude: place.lat,
-      city: place.city,
-      country: place.country,
-      source: place.source,
-      geodata: place.geodata,
+      id:         place.id,
+      name:       place.name,
+      longitude:  place.lon,
+      latitude:   place.lat,
+      city:       place.city,
+      country:    place.country,
+      source:     place.source,
+      geodata:    place.geodata,
+      created_at: place.created_at,
+      updated_at: place.updated_at,
       reverse_geocoded_at: place.reverse_geocoded_at
     }
   end

app/serializers/api/place_serializer.rb

@@ -21,8 +21,6 @@ def call
         raise InvalidJobName, 'Invalid job name'
       end
 
-    points.find_each(batch_size: 1_000) do |point|
-      point.async_reverse_geocode
-    end
+    points.find_each(&:async_reverse_geocode)
   end
 end

app/services/jobs/create.rb

@@ -17,10 +17,19 @@ def call
       return
     end
 
-    first_place = reverse_geocoded_places.shift
+    places = reverse_geocoded_places
+    first_place = places.shift
     update_place(first_place)
 
-    reverse_geocoded_places.each { |reverse_geocoded_place| fetch_and_create_place(reverse_geocoded_place) }
+    # Extract all osm_ids for preloading
+    osm_ids = places.map { |place| place.data['properties']['osm_id'].to_s }
+
+    # Preload all existing places with these osm_ids in a single query
+    existing_places = Place.where("geodata->'properties'->>'osm_id' IN (?)", osm_ids)
+                           .index_by { |p| p.geodata.dig('properties', 'osm_id').to_s }
+
+    # Process with preloaded data
+    places.each { |reverse_geocoded_place| fetch_and_create_place(reverse_geocoded_place, existing_places) }
   end
 
   private
@@ -41,9 +50,9 @@ def update_place(reverse_geocoded_place)
     )
   end
 
-  def fetch_and_create_place(reverse_geocoded_place)
+  def fetch_and_create_place(reverse_geocoded_place, existing_places = nil)
     data = reverse_geocoded_place.data
-    new_place = find_place(data)
+    new_place = find_place(data, existing_places)
 
     new_place.name = place_name(data)
     new_place.city = data['properties']['city']
@@ -57,16 +66,17 @@ def fetch_and_create_place(reverse_geocoded_place)
     new_place.save!
   end
 
-  def reverse_geocoded?
-    place.geodata.present?
-  end
+  def find_place(place_data, existing_places = nil)
+    osm_id = place_data['properties']['osm_id'].to_s
 
-  def find_place(place_data)
-    found_place = Place.where(
-      "geodata->'properties'->>'osm_id' = ?", place_data['properties']['osm_id'].to_s
-    ).first
-
-    return found_place if found_place.present?
+    # Use the preloaded data if available
+    if existing_places
+      return existing_places[osm_id] if existing_places[osm_id].present?
+    else
+      # Fall back to individual query if no preloaded data
+      found_place = Place.where("geodata->'properties'->>'osm_id' = ?", osm_id).first
+      return found_place if found_place.present?
+    end
 
     Place.find_or_initialize_by(
       lonlat: "POINT(#{place_data['geometry']['coordinates'][0].to_f.round(5)} #{place_data['geometry']['coordinates'][1].to_f.round(5)})",

app/services/reverse_geocoding/places/fetch_data.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+class Trips::Countries
+  FILE_PATH = Rails.root.join('lib/assets/countries.json')
+
+  def initialize(trip, batch_count = 2)
+    @trip = trip
+    @batch_count = batch_count
+    @factory = RGeo::Geographic.spherical_factory
+    @file = File.read(FILE_PATH)
+    @countries_features =
+      RGeo::GeoJSON.decode(@file, json_parser: :json, geo_factory: @factory)
+  end
+
+  def call
+    all_points = @trip.points.to_a
+    total_points = all_points.size
+
+    # Return empty hash if no points
+    return {} if total_points.zero?
+
+    batches = split_into_batches(all_points, @batch_count)
+    threads_results = process_batches_in_threads(batches, total_points)
+
+    merge_thread_results(threads_results).uniq.compact
+  end
+
+  private
+
+  def split_into_batches(points, batch_count)
+    batch_count = [batch_count, 1].max # Ensure batch_count is at least 1
+    batch_size = (points.size / batch_count.to_f).ceil
+    points.each_slice(batch_size).to_a
+  end
+
+  def process_batches_in_threads(batches, total_points)
+    threads_results = []
+    threads = []
+
+    batches.each do |batch|
+      threads << Thread.new do
+        threads_results << process_batch(batch)
+      end
+    end
+
+    threads.each(&:join)
+    threads_results
+  end
+
+  def merge_thread_results(threads_results)
+    countries = []
+
+    threads_results.each do |result|
+      countries.concat(result)
+    end
+
+    countries
+  end
+
+  def process_batch(points)
+    points.map do |point|
+      country_code = geocode_point(point)
+      next unless country_code
+
+      country_code
+    end
+  end
+
+  def geocode_point(point)
+    lonlat = point.lonlat
+    return nil unless lonlat
+
+    latitude = lonlat.y
+    longitude = lonlat.x
+
+    fetch_country_code(latitude, longitude)
+  end
+
+  def fetch_country_code(latitude, longitude)
+    results = Geocoder.search([latitude, longitude], limit: 1)
+    return nil unless results.any?
+
+    result = results.first
+    result.data['properties']['countrycode']
+  rescue StandardError => e
+    Rails.logger.error("Error geocoding point: #{e.message}")
+
+    ExceptionReporter.call(e)
+
+    nil
+  end
+end