Handle errors in feeds and set last_scraped_at properly

app/avo/resources/feed.rb

@@ -8,10 +8,10 @@ class Avo::Resources::Feed < Avo::BaseResource
   def fields
     field :id, as: :id
     field :title, as: :text
+    field :last_scraped_at, as: :date_time, sortable: true
     field :description, as: :textarea
     field :language, as: :text
     field :url, as: :text
-    field :last_scraped, as: :date_time, sortable: true
     field :raw, as: :code
     field :government, as: :belongs_to
 

app/avo/resources/promise.rb

@@ -51,7 +51,7 @@ def fields
     # field :keywords_enrichment_status, as: :text
     # field :keywords_extracted_at, as: :date_time
     # field :last_enrichment_at, as: :date_time
-    field :last_evidence_date, as: :date_time
+    field :last_evidence_date, as: :date_time, sortable: true
     # field :last_progress_update_at, as: :date_time
     # field :last_scored_at, as: :date_time
     # field :last_updated_at, as: :date_time

app/controllers/feeds_controller.rb

@@ -18,9 +18,4 @@ def show
     def set_feed
       @feed = Feed.find(params.expect(:id))
     end
-
-    # Only allow a list of trusted parameters through.
-    def feed_params
-      params.expect(feed: [ :title, :description, :language, :url, :last_scraped, :raw, :government_id ])
-    end
 end

app/models/feed.rb

@@ -15,13 +15,27 @@ def refresh!
     source_url = raw_feed.url
 
     self.raw = JSON.parse(raw_feed.to_json).except("entries")
+    self.last_scraped_at = Time.now
+    self.last_scrape_failed_at = nil
+    self.error_message = nil
+
+    self.save!
 
     create_entries!(raw_feed.entries)
+  rescue => e
+    Rails.logger.error("Error refreshing feed: #{e.message}")
+    self.last_scrape_failed_at = Time.now
+    self.error_message = e.message
+    self.save!
   end
 
   def create_entries!(entries)
     entries.each do |entry|
       raise "URL is required" unless entry.url.present?
+      if Entry.where(url: entry.url).where.not(feed: self).exists?
+        Rails.logger.info("Entry already exists for URL: #{entry.url} from a different feed, skipping")
+        next
+      end
       Entry.find_or_create_by!(government: government, feed: self, url: entry.url) do |rec|
         rec.title = entry.title
         rec.summary = entry.summary

db/migrate/20250623150313_add_error_to_feeds.rb

@@ -0,0 +1,8 @@
+class AddErrorToFeeds < ActiveRecord::Migration[8.0]
+  def change
+    remove_column :feeds, :last_scraped, :timestamp
+    add_column :feeds, :last_scraped_at, :timestamp
+    add_column :feeds, :error_message, :string
+    add_column :feeds, :last_scrape_failed_at, :timestamp
+  end
+end