diff --git a/app/avo/resources/feed.rb b/app/avo/resources/feed.rb index b20f81c..c1204cd 100644 --- a/app/avo/resources/feed.rb +++ b/app/avo/resources/feed.rb @@ -8,10 +8,10 @@ class Avo::Resources::Feed < Avo::BaseResource def fields field :id, as: :id field :title, as: :text + field :last_scraped_at, as: :date_time, sortable: true field :description, as: :textarea field :language, as: :text field :url, as: :text - field :last_scraped, as: :date_time, sortable: true field :raw, as: :code field :government, as: :belongs_to diff --git a/app/avo/resources/promise.rb b/app/avo/resources/promise.rb index 28a3b40..3e883a9 100644 --- a/app/avo/resources/promise.rb +++ b/app/avo/resources/promise.rb @@ -51,7 +51,7 @@ def fields # field :keywords_enrichment_status, as: :text # field :keywords_extracted_at, as: :date_time # field :last_enrichment_at, as: :date_time - field :last_evidence_date, as: :date_time + field :last_evidence_date, as: :date_time, sortable: true # field :last_progress_update_at, as: :date_time # field :last_scored_at, as: :date_time # field :last_updated_at, as: :date_time diff --git a/app/controllers/feeds_controller.rb b/app/controllers/feeds_controller.rb index 6b7da24..6337a5a 100644 --- a/app/controllers/feeds_controller.rb +++ b/app/controllers/feeds_controller.rb @@ -18,9 +18,4 @@ def show def set_feed @feed = Feed.find(params.expect(:id)) end - - # Only allow a list of trusted parameters through. - def feed_params - params.expect(feed: [ :title, :description, :language, :url, :last_scraped, :raw, :government_id ]) - end end diff --git a/app/models/feed.rb b/app/models/feed.rb index 2dcef30..9a30554 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -15,13 +15,27 @@ def refresh! source_url = raw_feed.url self.raw = JSON.parse(raw_feed.to_json).except("entries") + self.last_scraped_at = Time.now + self.last_scrape_failed_at = nil + self.error_message = nil + + self.save! create_entries!(raw_feed.entries) + rescue => e + Rails.logger.error("Error refreshing feed: #{e.message}") + self.last_scrape_failed_at = Time.now + self.error_message = e.message + self.save! end def create_entries!(entries) entries.each do |entry| raise "URL is required" unless entry.url.present? + if Entry.where(url: entry.url).where.not(feed: self).exists? + Rails.logger.info("Entry already exists for URL: #{entry.url} from a different feed, skipping") + next + end Entry.find_or_create_by!(government: government, feed: self, url: entry.url) do |rec| rec.title = entry.title rec.summary = entry.summary diff --git a/db/migrate/20250623150313_add_error_to_feeds.rb b/db/migrate/20250623150313_add_error_to_feeds.rb new file mode 100644 index 0000000..6acba93 --- /dev/null +++ b/db/migrate/20250623150313_add_error_to_feeds.rb @@ -0,0 +1,8 @@ +class AddErrorToFeeds < ActiveRecord::Migration[8.0] + def change + remove_column :feeds, :last_scraped, :timestamp + add_column :feeds, :last_scraped_at, :timestamp + add_column :feeds, :error_message, :string + add_column :feeds, :last_scrape_failed_at, :timestamp + end +end diff --git a/db/schema.rb b/db/schema.rb index 7fbfd8d..9bd21ca 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2025_06_20_152703) do +ActiveRecord::Schema[8.0].define(version: 2025_06_23_150313) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" @@ -139,11 +139,13 @@ t.text "description" t.string "language" t.string "url" - t.datetime "last_scraped", precision: nil t.jsonb "raw" t.bigint "government_id", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.datetime "last_scraped_at", precision: nil + t.string "error_message" + t.datetime "last_scrape_failed_at", precision: nil t.index ["government_id"], name: "index_feeds_on_government_id" end