From 9a11bd0125f0c80b4bc553da3f66f19c6d6c6d36 Mon Sep 17 00:00:00 2001 From: panioglo Date: Thu, 18 Aug 2016 15:39:07 +0300 Subject: [PATCH 01/12] added more sections for aggregations --- app/models/grouped_issue.rb | 2 +- app/models/issue.rb | 68 +++++++++++++++++++++++++++++---- app/views/errors/show.html.haml | 7 ++++ 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/app/models/grouped_issue.rb b/app/models/grouped_issue.rb index 3f321864..f13c9a76 100644 --- a/app/models/grouped_issue.rb +++ b/app/models/grouped_issue.rb @@ -25,7 +25,7 @@ def aggregations (attribute) if found data[found]["count"] += 1 else - data.push( { attribute => value, "count" => 0, "created_at" => issue.created_at, "updated_at" => issue.updated_at } ) + data.push( { attribute => value, "count" => 1, "created_at" => issue.created_at, "updated_at" => issue.updated_at } ) end end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 3e233d67..df78874e 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -40,7 +40,7 @@ def breadcrumbs_stacktrace def http_data(key = nil) all_data = get_interfaces(:http).try(:_data) - return false if all_data.blank? + return nil if all_data.blank? return all_data if key.nil? all_data[key] end @@ -57,7 +57,7 @@ def get_platform_frames end def get_frames(frame = nil) - frames = get_platform_frames.first + frames = get_platform_frames.try(:first) return frames if frame.nil? frames._data[frame] end @@ -66,18 +66,72 @@ def environment error.data[:environment] end - def browser - headers = error.data.try(:[], :interfaces).try(:[], :http).try(:[], :headers) - unless headers.nil? - headers.each do |hash| - return UserAgent.parse(hash[:user_agent]).browser if hash[:user_agent] + def version + modules = error.data[:modules] + unless modules.nil? + case self.platform + when 'ruby' + key = 'rails' end + "#{key.capitalize}/#{modules[key.to_sym]}" if defined? key end rescue => e Raven.capture_exception(e) 'Could not parse data!' end + def notifier_remote_address + http_data(:env).try(:[], :REMOTE_ADDR) + rescue => e + Raven.capture_exception(e) + 'Could not parse data!' + end + + def server_hostnames + error.data.try(:[], :server_name) + rescue => e + Raven.capture_exception(e) + 'Could not parse data!' + end + + def file + get_frames.try(:get_culprit_string, with_lineno: get_frames.try(:_data).try(:[], :lineno).present?) + rescue => e + Raven.capture_exception(e) + 'Could not parse data!' + end + + def url + url_string = http_data(:url) + rescue => e + Raven.capture_exception(e) + 'Could not parse data!' + end + + def browser_platform + user_agent = get_headers(:user_agent) + return UserAgent.parse(user_agent).platform unless user_agent.nil? + rescue => e + Raven.capture_exception(e) + 'Could not parse data!' + end + + def browser + user_agent = get_headers(:user_agent) + return UserAgent.parse(user_agent).browser unless user_agent.nil? + rescue => e + Raven.capture_exception(e) + 'Could not parse data!' + end + + def user + user_data = get_interfaces(:user) + return "##{user_data._data[:id]} #{user_data._data[:user_name]} #{user_data._data[:email]}" unless user_data.nil? + rescue => e + Raven.capture_exception(e) + 'Could not parse data!' + end + def self.more_than_10_errors(member) GroupedIssueMailer.more_than_10_errors(member).deliver_later end diff --git a/app/views/errors/show.html.haml b/app/views/errors/show.html.haml index 39c34abf..4db63425 100644 --- a/app/views/errors/show.html.haml +++ b/app/views/errors/show.html.haml @@ -46,3 +46,10 @@ // use secondary if the returned item is a hash = render partial: 'errors/aggregations', locals: { title: 'Subscribers', attribute: "subscriber", secondary: "name", data: @error.aggregations("subscriber"), panel: "Two" } = render partial: 'errors/aggregations', locals: { title: 'Browsers', attribute: "browser", data: @error.aggregations("browser"), panel: "Three" } + = render partial: 'errors/aggregations', locals: { title: 'Browser platform', attribute: "browser_platform", data: @error.aggregations("browser_platform"), panel: "Four" } + = render partial: 'errors/aggregations', locals: { title: 'Users', attribute: "user", data: @error.aggregations("user"), panel: "Five" } + = render partial: 'errors/aggregations', locals: { title: 'URLs', attribute: "url", data: @error.aggregations("url"), panel: "Six" } + = render partial: 'errors/aggregations', locals: { title: 'Versions', attribute: "version", data: @error.aggregations("version"), panel: "Seven" } + = render partial: 'errors/aggregations', locals: { title: 'Files', attribute: "file", data: @error.aggregations("file"), panel: "Eight" } + = render partial: 'errors/aggregations', locals: { title: 'Server hostnames', attribute: "server_hostnames", data: @error.aggregations("server_hostnames"), panel: "Nine" } + = render partial: 'errors/aggregations', locals: { title: 'Notifier remote address', attribute: "notifier_remote_address", data: @error.aggregations("notifier_remote_address"), panel: "Ten" } From 4e4a6de02145681f3d72631bfc68521db51c80bf Mon Sep 17 00:00:00 2001 From: panioglo Date: Thu, 18 Aug 2016 15:42:41 +0300 Subject: [PATCH 02/12] humanize instead op capitalizing string attribute --- app/views/errors/_aggregations.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/errors/_aggregations.html.haml b/app/views/errors/_aggregations.html.haml index efbcbdfb..95417e14 100644 --- a/app/views/errors/_aggregations.html.haml +++ b/app/views/errors/_aggregations.html.haml @@ -7,7 +7,7 @@ %table.table#members-container %thead %tr - %th=attribute.capitalize + %th=attribute.humanize %th # of notices %th First seen at %th Last seen at From 1deeeb59026106defc1da98ac910e6814531da8a Mon Sep 17 00:00:00 2001 From: panioglo Date: Fri, 19 Aug 2016 09:44:44 +0300 Subject: [PATCH 03/12] add aggregates table --- db/migrate/20160818125259_create_aggregations.rb | 11 +++++++++++ db/schema.rb | 10 +++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20160818125259_create_aggregations.rb diff --git a/db/migrate/20160818125259_create_aggregations.rb b/db/migrate/20160818125259_create_aggregations.rb new file mode 100644 index 00000000..8aac0b12 --- /dev/null +++ b/db/migrate/20160818125259_create_aggregations.rb @@ -0,0 +1,11 @@ +class CreateAggregations < ActiveRecord::Migration + def change + create_table :aggregates do |t| + t.integer :group_id + t.string :type + t.jsonb :value, default: {} + + t.timestamps null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 5aa020af..6f83266d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20160808131218) do +ActiveRecord::Schema.define(version: 20160818125259) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -28,6 +28,14 @@ t.datetime "updated_at" end + create_table "aggregates", force: :cascade do |t| + t.integer "group_id" + t.string "type" + t.jsonb "value", :default=>{} + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false + end + create_table "websites", force: :cascade do |t| t.string "title", :null=>false t.string "domain", :null=>false From aeefe6e6cb8c2276733476041cec47cfe5ce8d7c Mon Sep 17 00:00:00 2001 From: panioglo Date: Fri, 19 Aug 2016 09:46:51 +0300 Subject: [PATCH 04/12] model dependencies --- app/models/aggregate.rb | 3 +++ app/models/grouped_issue.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 app/models/aggregate.rb diff --git a/app/models/aggregate.rb b/app/models/aggregate.rb new file mode 100644 index 00000000..ce21a35b --- /dev/null +++ b/app/models/aggregate.rb @@ -0,0 +1,3 @@ +class Aggregate < ActiveRecord::Base + belongs_to :group, class_name: 'GroupedIssue', foreign_key: group_id +end \ No newline at end of file diff --git a/app/models/grouped_issue.rb b/app/models/grouped_issue.rb index f13c9a76..9191c741 100644 --- a/app/models/grouped_issue.rb +++ b/app/models/grouped_issue.rb @@ -6,8 +6,8 @@ class GroupedIssue < ActiveRecord::Base has_many :subscribers, -> { uniq }, through: :issues, foreign_key: 'group_id' has_many :issues, foreign_key: 'group_id', dependent: :destroy has_many :messages, through: :issues + has_many :aggregates, dependent: :destroy enumerize :level, in: [:debug, :error, :fatal, :info, :warning], default: :error - # enumerize :issue_logger, in: { javascript: 1, php: 2 }, default: :javascript enumerize :status, in: { muted: 1, resolved: 2, unresolved: 3 }, default: :unresolved, predicates: true, scope: true friendly_id :message, use: :slugged before_save :check_fields From 84ee6ae5fe31c078c17f5746ce2e4be4cd016e55 Mon Sep 17 00:00:00 2001 From: panioglo Date: Fri, 19 Aug 2016 12:03:24 +0300 Subject: [PATCH 05/12] updated aggregates table to not use restricted words --- db/migrate/20160818125259_create_aggregations.rb | 4 ++-- db/schema.rb | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/db/migrate/20160818125259_create_aggregations.rb b/db/migrate/20160818125259_create_aggregations.rb index 8aac0b12..a82048ae 100644 --- a/db/migrate/20160818125259_create_aggregations.rb +++ b/db/migrate/20160818125259_create_aggregations.rb @@ -1,8 +1,8 @@ class CreateAggregations < ActiveRecord::Migration def change create_table :aggregates do |t| - t.integer :group_id - t.string :type + t.belongs_to :grouped_issue, index: true, foreign_key: { references: :grouped_issues, on_update: :restrict, on_delete: :cascade } + t.string :agg_type t.jsonb :value, default: {} t.timestamps null: false diff --git a/db/schema.rb b/db/schema.rb index 6f83266d..00dd2dfd 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -28,14 +28,6 @@ t.datetime "updated_at" end - create_table "aggregates", force: :cascade do |t| - t.integer "group_id" - t.string "type" - t.jsonb "value", :default=>{} - t.datetime "created_at", :null=>false - t.datetime "updated_at", :null=>false - end - create_table "websites", force: :cascade do |t| t.string "title", :null=>false t.string "domain", :null=>false @@ -81,6 +73,14 @@ end add_index "grouped_issues", ["website_id", "checksum"], :name=>"index_grouped_issues_on_website_id_and_checksum", :unique=>true + create_table "aggregates", force: :cascade do |t| + t.integer "grouped_issue_id", :index=>{:name=>"index_aggregates_on_grouped_issue_id"}, :foreign_key=>{:references=>"grouped_issues", :name=>"fk_aggregates_grouped_issue_id", :on_update=>:restrict, :on_delete=>:cascade} + t.string "agg_type" + t.jsonb "value", :default=>{} + t.datetime "created_at", :null=>false + t.datetime "updated_at", :null=>false + end + create_table "integrations", force: :cascade do |t| t.integer "website_id", :index=>{:name=>"index_integrations_on_website_id"}, :foreign_key=>{:references=>"websites", :name=>"fk_integrations_website_id", :on_update=>:restrict, :on_delete=>:cascade} t.string "provider", :null=>false From efbf6a0bea51ad711e5c49be0daa8bac953526c5 Mon Sep 17 00:00:00 2001 From: panioglo Date: Fri, 19 Aug 2016 14:50:29 +0300 Subject: [PATCH 06/12] updates for the aggregates section --- app/models/aggregate.rb | 2 +- app/models/grouped_issue.rb | 18 +------------ app/models/issue.rb | 5 ++++ app/views/errors/_aggregations.html.haml | 24 ++++++++---------- app/views/errors/show.html.haml | 14 +++-------- lib/error_store.rb | 32 ++++++++++++++++++++++++ 6 files changed, 52 insertions(+), 43 deletions(-) diff --git a/app/models/aggregate.rb b/app/models/aggregate.rb index ce21a35b..4df88fd8 100644 --- a/app/models/aggregate.rb +++ b/app/models/aggregate.rb @@ -1,3 +1,3 @@ class Aggregate < ActiveRecord::Base - belongs_to :group, class_name: 'GroupedIssue', foreign_key: group_id + belongs_to :grouped_issue end \ No newline at end of file diff --git a/app/models/grouped_issue.rb b/app/models/grouped_issue.rb index 9191c741..afc0c92d 100644 --- a/app/models/grouped_issue.rb +++ b/app/models/grouped_issue.rb @@ -6,7 +6,7 @@ class GroupedIssue < ActiveRecord::Base has_many :subscribers, -> { uniq }, through: :issues, foreign_key: 'group_id' has_many :issues, foreign_key: 'group_id', dependent: :destroy has_many :messages, through: :issues - has_many :aggregates, dependent: :destroy + has_many :aggregates enumerize :level, in: [:debug, :error, :fatal, :info, :warning], default: :error enumerize :status, in: { muted: 1, resolved: 2, unresolved: 3 }, default: :unresolved, predicates: true, scope: true friendly_id :message, use: :slugged @@ -16,22 +16,6 @@ def users_affected subscribers.count end - def aggregations (attribute) - data = [] - self.issues.each do |issue| - value = issue.public_send(attribute) - unless value.nil? - found = data.index { |x| x[attribute] == value } - if found - data[found]["count"] += 1 - else - data.push( { attribute => value, "count" => 1, "created_at" => issue.created_at, "updated_at" => issue.updated_at } ) - end - end - end - data - end - private def check_fields diff --git a/app/models/issue.rb b/app/models/issue.rb index df78874e..18b7fa20 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -7,6 +7,7 @@ class Issue < ActiveRecord::Base accepts_nested_attributes_for :messages validates :message, presence: true after_create :issue_created + after_commit :increment_aggregate def error ErrorStore.find(self) @@ -20,6 +21,10 @@ def data decode_and_decompress(super) end + def increment_aggregate + ErrorStore::Aggregates.new.handle_aggregates(self) + end + def get_interfaces(interface = nil) all_interaces = error._get_interfaces return all_interaces if interface.nil? diff --git a/app/views/errors/_aggregations.html.haml b/app/views/errors/_aggregations.html.haml index 95417e14..15292e4e 100644 --- a/app/views/errors/_aggregations.html.haml +++ b/app/views/errors/_aggregations.html.haml @@ -1,28 +1,24 @@ .panel.panel-default .panel-heading - %a{"data-parent" => "#accordion", "data-toggle" => "collapse", :href => "#collapse#{panel}"} - %h4.panel-title=title - .panel-collapse.collapse{ id: "collapse#{panel}", class: "#{panel == 'One' ? 'in' : ''}" } + %a{"data-parent" => "#accordion", "data-toggle" => "collapse", :href => "#collapse#{data.agg_type}"} + %h4.panel-title=data.agg_type.humanize + .panel-collapse.collapse{ id: "collapse#{data.agg_type}", class: "#{data.agg_type == 'One' ? 'in' : ''}" } .panel-body %table.table#members-container %thead %tr - %th=attribute.humanize + %th=data.agg_type.humanize %th # of notices %th First seen at %th Last seen at %tbody.websites - - data.each do |element| + - data.value.keys.each do |key| %tr - - if defined?(secondary) - %td - =element[attribute][secondary] - - else - %td - =element[attribute] %td - =element["count"] + =key %td - =element["created_at"].strftime("%Y-%m-%d %l:%M") + =data.value[key]["count"] %td - =element["updated_at"].strftime("%Y-%m-%d %l:%M") + =data.value[key]["created_at"].to_time.strftime("%Y-%m-%d %l:%M") + %td + =data.value[key]["updated_at"].to_time.strftime("%Y-%m-%d %l:%M") diff --git a/app/views/errors/show.html.haml b/app/views/errors/show.html.haml index 4db63425..7894747d 100644 --- a/app/views/errors/show.html.haml +++ b/app/views/errors/show.html.haml @@ -42,14 +42,6 @@ #aggregations.tab-pane{ class: "#{params[:current_tab] == 'aggregations' ? 'active' : ''}" } - if params[:current_tab] == 'aggregations' #accordion.panel-group - = render partial: 'errors/aggregations', locals: { title: 'Messages', attribute: "message", data: @error.aggregations("message"), panel: "One" } - // use secondary if the returned item is a hash - = render partial: 'errors/aggregations', locals: { title: 'Subscribers', attribute: "subscriber", secondary: "name", data: @error.aggregations("subscriber"), panel: "Two" } - = render partial: 'errors/aggregations', locals: { title: 'Browsers', attribute: "browser", data: @error.aggregations("browser"), panel: "Three" } - = render partial: 'errors/aggregations', locals: { title: 'Browser platform', attribute: "browser_platform", data: @error.aggregations("browser_platform"), panel: "Four" } - = render partial: 'errors/aggregations', locals: { title: 'Users', attribute: "user", data: @error.aggregations("user"), panel: "Five" } - = render partial: 'errors/aggregations', locals: { title: 'URLs', attribute: "url", data: @error.aggregations("url"), panel: "Six" } - = render partial: 'errors/aggregations', locals: { title: 'Versions', attribute: "version", data: @error.aggregations("version"), panel: "Seven" } - = render partial: 'errors/aggregations', locals: { title: 'Files', attribute: "file", data: @error.aggregations("file"), panel: "Eight" } - = render partial: 'errors/aggregations', locals: { title: 'Server hostnames', attribute: "server_hostnames", data: @error.aggregations("server_hostnames"), panel: "Nine" } - = render partial: 'errors/aggregations', locals: { title: 'Notifier remote address', attribute: "notifier_remote_address", data: @error.aggregations("notifier_remote_address"), panel: "Ten" } + - unless @error.aggregates.blank? + - @error.aggregates.each do |record| + = render partial: 'errors/aggregations', locals: { data: record} diff --git a/lib/error_store.rb b/lib/error_store.rb index 3e92337a..d91869e1 100644 --- a/lib/error_store.rb +++ b/lib/error_store.rb @@ -149,6 +149,38 @@ def message end end + class Aggregates < StoreError + + ATTRIBUTES = [ + 'message', + 'subscriber', + 'browser', + 'browser_platform', + 'user', + 'url', + 'version', + 'file', + 'server_hostnames', + 'notifier_remote_address' + ] + + def handle_aggregates(issue) + ATTRIBUTES.each do |attribute| + unless (data = issue.public_send(attribute)).nil? + record = issue.group.aggregates.find_or_create_by(agg_type: attribute) + data = data.name if attribute == 'subscriber' + if record.value[data].nil? + record.value[data] = { :count => 1, :created_at => issue.created_at, :updated_at => issue.updated_at } + else + record.value[data]['count'] += 1 + record.value[data]['updated_at'] = issue.updated_at + end + record.save + end + end + end + end + # an exception raised if the user does not send the right credentials class MissingCredentials < StoreError; end # an exception raised if the website is missing From 2dd3c3300a0b55163c8eb6478d23e00b02769d4c Mon Sep 17 00:00:00 2001 From: panioglo Date: Fri, 19 Aug 2016 16:12:04 +0300 Subject: [PATCH 07/12] pull request fixes --- app/models/issue.rb | 2 +- app/views/errors/_aggregations.html.haml | 8 ++-- app/views/errors/show.html.haml | 4 +- .../20160818125259_create_aggregations.rb | 2 +- db/schema.rb | 2 +- lib/error_store.rb | 32 ---------------- lib/error_store/aggregates.rb | 37 +++++++++++++++++++ 7 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 lib/error_store/aggregates.rb diff --git a/app/models/issue.rb b/app/models/issue.rb index 18b7fa20..4fae4947 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -22,7 +22,7 @@ def data end def increment_aggregate - ErrorStore::Aggregates.new.handle_aggregates(self) + ErrorStore::Aggregates.new(self).handle_aggregates end def get_interfaces(interface = nil) diff --git a/app/views/errors/_aggregations.html.haml b/app/views/errors/_aggregations.html.haml index 15292e4e..2cf6bdae 100644 --- a/app/views/errors/_aggregations.html.haml +++ b/app/views/errors/_aggregations.html.haml @@ -1,13 +1,13 @@ .panel.panel-default .panel-heading - %a{"data-parent" => "#accordion", "data-toggle" => "collapse", :href => "#collapse#{data.agg_type}"} - %h4.panel-title=data.agg_type.humanize - .panel-collapse.collapse{ id: "collapse#{data.agg_type}", class: "#{data.agg_type == 'One' ? 'in' : ''}" } + %a{"data-parent" => "#accordion", "data-toggle" => "collapse", :href => "#collapse#{data.name}"} + %h4.panel-title=data.name.humanize + .panel-collapse.collapse{ id: "collapse#{data.name}", class: "#{data.name == @error.aggregates[0].name ? 'in' : ''}" } .panel-body %table.table#members-container %thead %tr - %th=data.agg_type.humanize + %th=data.name.humanize %th # of notices %th First seen at %th Last seen at diff --git a/app/views/errors/show.html.haml b/app/views/errors/show.html.haml index 7894747d..ee5db541 100644 --- a/app/views/errors/show.html.haml +++ b/app/views/errors/show.html.haml @@ -43,5 +43,5 @@ - if params[:current_tab] == 'aggregations' #accordion.panel-group - unless @error.aggregates.blank? - - @error.aggregates.each do |record| - = render partial: 'errors/aggregations', locals: { data: record} + = render partial: 'errors/aggregations', collection: @error.aggregates, as: :data +d \ No newline at end of file diff --git a/db/migrate/20160818125259_create_aggregations.rb b/db/migrate/20160818125259_create_aggregations.rb index a82048ae..abca7c16 100644 --- a/db/migrate/20160818125259_create_aggregations.rb +++ b/db/migrate/20160818125259_create_aggregations.rb @@ -2,7 +2,7 @@ class CreateAggregations < ActiveRecord::Migration def change create_table :aggregates do |t| t.belongs_to :grouped_issue, index: true, foreign_key: { references: :grouped_issues, on_update: :restrict, on_delete: :cascade } - t.string :agg_type + t.string :name t.jsonb :value, default: {} t.timestamps null: false diff --git a/db/schema.rb b/db/schema.rb index 00dd2dfd..3f0a8230 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -75,7 +75,7 @@ create_table "aggregates", force: :cascade do |t| t.integer "grouped_issue_id", :index=>{:name=>"index_aggregates_on_grouped_issue_id"}, :foreign_key=>{:references=>"grouped_issues", :name=>"fk_aggregates_grouped_issue_id", :on_update=>:restrict, :on_delete=>:cascade} - t.string "agg_type" + t.string "name" t.jsonb "value", :default=>{} t.datetime "created_at", :null=>false t.datetime "updated_at", :null=>false diff --git a/lib/error_store.rb b/lib/error_store.rb index d91869e1..3e92337a 100644 --- a/lib/error_store.rb +++ b/lib/error_store.rb @@ -149,38 +149,6 @@ def message end end - class Aggregates < StoreError - - ATTRIBUTES = [ - 'message', - 'subscriber', - 'browser', - 'browser_platform', - 'user', - 'url', - 'version', - 'file', - 'server_hostnames', - 'notifier_remote_address' - ] - - def handle_aggregates(issue) - ATTRIBUTES.each do |attribute| - unless (data = issue.public_send(attribute)).nil? - record = issue.group.aggregates.find_or_create_by(agg_type: attribute) - data = data.name if attribute == 'subscriber' - if record.value[data].nil? - record.value[data] = { :count => 1, :created_at => issue.created_at, :updated_at => issue.updated_at } - else - record.value[data]['count'] += 1 - record.value[data]['updated_at'] = issue.updated_at - end - record.save - end - end - end - end - # an exception raised if the user does not send the right credentials class MissingCredentials < StoreError; end # an exception raised if the website is missing diff --git a/lib/error_store/aggregates.rb b/lib/error_store/aggregates.rb new file mode 100644 index 00000000..d39504c4 --- /dev/null +++ b/lib/error_store/aggregates.rb @@ -0,0 +1,37 @@ +module ErrorStore + class Aggregates < StoreError + + def initialize(issue) + @issue = issue + end + + ATTRIBUTES = [ + 'message', + 'subscriber', + 'browser', + 'browser_platform', + 'user', + 'url', + 'version', + 'file', + 'server_hostnames', + 'notifier_remote_address' + ] + + def handle_aggregates + ATTRIBUTES.each do |attribute| + unless (data = @issue.public_send(attribute)).nil? + record = @issue.group.aggregates.find_or_create_by(name: attribute) + data = data.name if attribute == 'subscriber' + if record.value[data].nil? + record.value[data] = { :count => 1, :created_at => @issue.created_at, :updated_at => @issue.updated_at } + else + record.value[data]['count'] += 1 + record.value[data]['updated_at'] = @issue.updated_at + end + record.save + end + end + end + end +end \ No newline at end of file From 994b059bcd236d3541aa6144a378b84cefccc436 Mon Sep 17 00:00:00 2001 From: panioglo Date: Mon, 22 Aug 2016 08:59:24 +0300 Subject: [PATCH 08/12] run handle_aggregates in a backgroud job --- app/models/issue.rb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/app/models/issue.rb b/app/models/issue.rb index 4fae4947..0b480664 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -7,7 +7,7 @@ class Issue < ActiveRecord::Base accepts_nested_attributes_for :messages validates :message, presence: true after_create :issue_created - after_commit :increment_aggregate + after_commit :refresh_aggregates def error ErrorStore.find(self) @@ -21,8 +21,10 @@ def data decode_and_decompress(super) end - def increment_aggregate - ErrorStore::Aggregates.new(self).handle_aggregates + def refresh_aggregates + cache_key = self.object_id.to_s + Rails.cache.write(cache_key, self) + AggregatesWorker.perform_async(cache_key) end def get_interfaces(interface = nil) @@ -146,4 +148,12 @@ def issue_created GroupedIssueMailer.error_occurred(self, member).deliver_later end end + + class AggregatesWorker + include Sidekiq::Worker + def perform(cache_key) + record = Rails.cache.read(cache_key) + ErrorStore::Aggregates.new(record).handle_aggregates + end + end end From e80813fd6ba1a7cae4acefc466577707b9169b62 Mon Sep 17 00:00:00 2001 From: panioglo Date: Mon, 22 Aug 2016 09:00:01 +0300 Subject: [PATCH 09/12] fixed bug with aggregates --- lib/error_store/aggregates.rb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/error_store/aggregates.rb b/lib/error_store/aggregates.rb index d39504c4..b2ac810a 100644 --- a/lib/error_store/aggregates.rb +++ b/lib/error_store/aggregates.rb @@ -19,17 +19,19 @@ def initialize(issue) ] def handle_aggregates - ATTRIBUTES.each do |attribute| - unless (data = @issue.public_send(attribute)).nil? - record = @issue.group.aggregates.find_or_create_by(name: attribute) - data = data.name if attribute == 'subscriber' - if record.value[data].nil? - record.value[data] = { :count => 1, :created_at => @issue.created_at, :updated_at => @issue.updated_at } - else - record.value[data]['count'] += 1 - record.value[data]['updated_at'] = @issue.updated_at + unless @issue.nil? + ATTRIBUTES.each do |attribute| + unless (data = @issue.public_send(attribute)).nil? + record = @issue.group.aggregates.find_or_create_by(name: attribute) + data = data.name if attribute == 'subscriber' + if record.value[data].nil? + record.value[data] = { :count => 1, :created_at => @issue.created_at, :updated_at => @issue.updated_at } + else + record.value[data]['count'] += 1 + record.value[data]['updated_at'] = @issue.updated_at + end + record.save end - record.save end end end From b402073ead5e6a9f2f6f9aa6706cf7654ea91ee5 Mon Sep 17 00:00:00 2001 From: panioglo Date: Mon, 22 Aug 2016 15:16:25 +0300 Subject: [PATCH 10/12] send issue id in the perform action/ update specs for the issue model --- app/models/issue.rb | 8 +++----- spec/models/issue_spec.rb | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/app/models/issue.rb b/app/models/issue.rb index 0b480664..4e96edab 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -22,9 +22,7 @@ def data end def refresh_aggregates - cache_key = self.object_id.to_s - Rails.cache.write(cache_key, self) - AggregatesWorker.perform_async(cache_key) + AggregatesWorker.perform_async(self.id) end def get_interfaces(interface = nil) @@ -151,8 +149,8 @@ def issue_created class AggregatesWorker include Sidekiq::Worker - def perform(cache_key) - record = Rails.cache.read(cache_key) + def perform(issue_id) + record = Issue.find(issue_id) ErrorStore::Aggregates.new(record).handle_aggregates end end diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb index c59dd032..f1e09d3d 100644 --- a/spec/models/issue_spec.rb +++ b/spec/models/issue_spec.rb @@ -84,6 +84,26 @@ end end + describe 'refresh_aggregates' do + subject { issue.refresh_aggregates } + it 'calls perform' do + expect(Issue::AggregatesWorker).to receive(:perform_async) + subject + end + + it 'should push jobs to the queue' do + expect { + subject + }.to change(Issue::AggregatesWorker.jobs, :size).by(1) + subject + end + + it 'calls handle_aggregates' do + expect_any_instance_of(ErrorStore::Aggregates).to receive(:handle_aggregates) + Issue::AggregatesWorker.new.perform(issue.id) + end + end + describe "get_headers" do it 'returns all headers if no param' do expect(issue.get_headers).to eq(http[:headers]) @@ -96,11 +116,11 @@ context 'when http has no data' do before(:each) do issue.update_attributes(data: missing_http) end it 'should return false' do - expect(issue.get_headers).to eq(false) + expect(issue.get_headers).to be_nil end it 'should return false even if param is given' do - expect(issue.get_headers(:http)).to eq(false) + expect(issue.get_headers(:http)).to be_nil end end end @@ -117,11 +137,11 @@ context 'when http has no data' do before(:each) do issue.update_attributes(data: missing_http) end it 'should return false' do - expect(issue.http_data).to eq(false) + expect(issue.http_data).to be_nil end it 'should return false even if param is given' do - expect(issue.http_data(:headers)).to eq(false) + expect(issue.http_data(:headers)).to be_nil end end end From 09c7dba5dbca0d06e20beb1a513564fecfe10501 Mon Sep 17 00:00:00 2001 From: panioglo Date: Mon, 22 Aug 2016 16:00:10 +0300 Subject: [PATCH 11/12] specs for the aggregates class --- spec/lib/error_store/aggregates_spec.rb | 76 +++++++++++++++++++++++++ spec/models/grouped_issue_spec.rb | 33 +---------- 2 files changed, 78 insertions(+), 31 deletions(-) create mode 100644 spec/lib/error_store/aggregates_spec.rb diff --git a/spec/lib/error_store/aggregates_spec.rb b/spec/lib/error_store/aggregates_spec.rb new file mode 100644 index 00000000..4046273a --- /dev/null +++ b/spec/lib/error_store/aggregates_spec.rb @@ -0,0 +1,76 @@ +require 'rails_helper' + +RSpec.describe ErrorStore::Aggregates do + let(:user) { create :user } + let(:website) { create :website } + let!(:website_member) { create :website_member, website: website, user: user } + let!(:grouped_issue) { create :grouped_issue, website: website } + let(:subscriber) { create :subscriber, website: website } + let!(:issue_error) { create :issue, subscriber: subscriber, group: grouped_issue } + let(:mozilla_browser) { + '{"server_name":"sergiu-Lenovo-IdeaPad-Y510P","modules":{"rake":"10.4.2","i18n":"0.7.0","json":"1.8.3","minitest":"5.8.2","thread_safe":"0.3.5","tzinfo":"1.2.2","activesupport":"4.2.1","builder":"3.2.2","erubis":"2.7.0","mini_portile":"0.6.2","nokogiri":"1.6.6.2","rails-deprecated_sanitizer":"1.0.3","rails-dom-testing":"1.0.7","loofah":"2.0.3","rails-html-sanitizer":"1.0.2","actionview":"4.2.1","rack":"1.6.4","rack-test":"0.6.3","actionpack":"4.2.1","globalid":"0.3.6","activejob":"4.2.1","mime-types":"2.6.2","mail":"2.6.3","actionmailer":"4.2.1","activemodel":"4.2.1","arel":"6.0.3","activerecord":"4.2.1","debug_inspector":"0.0.2","binding_of_caller":"0.7.2","bundler":"1.11.2","coderay":"1.1.0","coffee-script-source":"1.10.0","execjs":"2.6.0","coffee-script":"2.4.1","thor":"0.19.1","railties":"4.2.1","coffee-rails":"4.1.0","multipart-post":"2.0.0","faraday":"0.9.2","multi_json":"1.11.2","jbuilder":"2.3.2","jquery-rails":"4.0.5","method_source":"0.8.2","slop":"3.6.0","pry":"0.10.3","sprockets":"3.4.0","sprockets-rails":"2.3.3","rails":"4.2.1","rdoc":"4.2.0","sass":"3.4.19","tilt":"2.0.1","sass-rails":"5.0.4","sdoc":"0.4.1","sentry-raven":"0.15.2","spring":"1.4.1","sqlite3":"1.3.11","turbolinks":"2.5.3","uglifier":"2.7.2","web-console":"2.2.1"},"extra":{},"tags":{},"errors":[{"type":"invalid_data","name":"timestamp","value":"2016-02-15T06:01:29"}],"interfaces":{"exception":{"values":[{"type":"ZeroDivisionError","value":"\"divided by 0\"","module":"","stacktrace":{"frames":[{"abs_path":"\/home\/sergiu\/.rvm\/rubies\/ruby-2.2.2\/lib\/ruby\/2.2.0\/webrick\/server.rb","filename":"webrick\/server.rb","function":"block in start_thread","context_line":" block ? block.call(sock) : run(sock)\n","pre_context":["module ActionController\n"," module ImplicitRender\n"," def send_action(method, *args)\n"],"post_context":[" default_render unless performed?\n"," ret\n"," end\n"],"lineno":4},{"abs_path":"\/home\/sergiu\/ravenapp\/app\/controllers\/home_controller.rb","filename":"app\/controllers\/home_controller.rb","function":"index","context_line":" 1\/0\n","pre_context":[" # Prevent CSRF attacks by raising an exception.\n"," # For APIs, you may want to use :null_session instead.\n"," def index\n"],"post_context":[" end\n","end\n",""],"lineno":5},{"abs_path":"\/home\/sergiu\/ravenapp\/app\/controllers\/home_controller.rb","filename":"app\/controllers\/home_controller.rb","function":"\/","context_line":" 1\/0\n","pre_context":[" # Prevent CSRF attacks by raising an exception.\n"," # For APIs, you may want to use :null_session instead.\n"," def index\n"],"post_context":[" end\n","end\n",""],"lineno":5}],"frames_omitted":null,"has_frames":true}}],"exc_omitted":null},"http":{"env":{"REMOTE_ADDR":"127.0.0.1","SERVER_NAME":"localhost","SERVER_PORT":"3001"},"headers":[{"host":"localhost:3001"},{"connection":"keep-alive"},{"accept":"text\/html,application\/xhtml+xml,application\/xml;q=0.9,image\/webp,*\/*;q=0.8"},{"upgrade_insecure_requests":"1"},{"user_agent":"Mozilla\/5.0 (X11; Linux x86_64) Gecko\/20100101 Firefox\/46.0"},{"accept_encoding":"gzip, deflate, sdch"},{"accept_language":"en-US,en;q=0.8"},{"cookie":"currentConfigName=%22default%22; pickedWebsite=1; _epiclogger_session=NTIwU2prYUd2T0dEd3FGQWE0WUFaL3RDY0huRGFnV1Z5TEhOQ3RtWUZZTTVRSzgvRTZvdEI2SFRsSVQ0ajVidHRWQnA5ck9wT3ZQU095N0dkWjEza0dpYWlmekxyRzFJbEFVNk5zRExmMzg3Q2c2MFBWdi9UYVUyWjdkWHVMNUFaWFJzZ2pEMGdwd3JJSGpSNjlEK2o3ZjlWZEhISEprK1hXOFNvaGdRdHg2MkFxN0lrcmlIdUtQazVWUjNGaWJvUGVYTHJncEc2OWhpaHBZbXNqcVhUcjM0ZWQ5bDFnWDBVSGlaOE5rdGxiOHNDU2NUS3BaSjd4eUZSRklzVnU5M3Z0TmJLUzF6ZWxjOGUrRmF2NkZ6ZCtGMUdoQVdFUSt0am9KT2lDODRMckJwbWQ1ZU5hV1hhZmt2bHdDZHZibEFmMExXNTI5Tmt..."},{"version":"HTTP\/1.1"}],"url":"http:\/\/localhost\/\/"}},"site":null,"environment":null,"version":"5"}' + } + let(:issue2) { create :issue, subscriber, group: grouped_issue, data: mozilla_browser } + + describe 'intialize' do + it 'assigns provided issue' do + expect( ErrorStore::Aggregates.new(issue_error).instance_variable_get(:@issue) ).to eq(issue_error) + end + end + + it 'ATTRIBUTES' do + expect(ErrorStore::Aggregates::ATTRIBUTES).to eq( + [ + 'message', + 'subscriber', + 'browser', + 'browser_platform', + 'user', + 'url', + 'version', + 'file', + 'server_hostnames', + 'notifier_remote_address' + ]) + end + + describe 'handle_aggregates' do + subject { ErrorStore::Aggregates.new(issue_error).handle_aggregates } + + it 'should return nil if no issue' do + expect{ErrorStore::Aggregates.new(nil).handle_aggregates}.not_to change{Aggregate.count} + end + + it 'should not create/update record if data is nil' do + subject + expect( Aggregate.find_by_name('user') ).to be_nil + end + + it 'should create aggregate if record not found' do + expect{ subject }.to change{ Aggregate.find_by_name('message') }.from(nil) + end + + it 'should update existing aggregate' do + record = grouped_issue.aggregates.create( name: 'message', value: { issue_error.message => { :count => 1, :created_at => issue_error.created_at, :updated_at => issue_error.updated_at } } ) + + expect{ + subject + record.reload + }.to change{ record.value[issue_error.message]["count"] }.from(1).to(2) + end + + it 'should update with a new value' do + record = grouped_issue.aggregates.create( name: 'message', value: { 'Not good' => { :count => 1, :created_at => issue_error.created_at, :updated_at => issue_error.updated_at } } ) + + expect{ + subject + record.reload + }.to change{ record.value.length }.from(1).to(2) + end + + it 'should increase the number of records' do + expect{ subject }.to change{ Aggregate.count } + end + + end +end \ No newline at end of file diff --git a/spec/models/grouped_issue_spec.rb b/spec/models/grouped_issue_spec.rb index f6053439..025e91b4 100644 --- a/spec/models/grouped_issue_spec.rb +++ b/spec/models/grouped_issue_spec.rb @@ -5,6 +5,8 @@ let(:website) { create :website } let!(:website_member) { create :website_member, website: website, user: user } let!(:grouped_issue) { create :grouped_issue, website: website } + let(:subscriber) { create :subscriber, website: website } + let!(:issue_error) { create :issue, subscriber: subscriber, group: grouped_issue } it { is_expected.to enumerize(:level).in(:debug, :error, :fatal, :info, :warning).with_default(:error) } it { is_expected.to enumerize(:status).in(:muted, :resolved, :unresolved) } @@ -23,35 +25,4 @@ it "has a valid factory" do expect(grouped_issue).to be_valid end - - context '#aggregations' do - let(:subscriber) { create :subscriber, website: website } - let!(:issue_error) { create :issue, subscriber: subscriber, group: grouped_issue } - it 'returns different browser occurences' do - mozilla_browser = '{"server_name":"sergiu-Lenovo-IdeaPad-Y510P","modules":{"rake":"10.4.2","i18n":"0.7.0","json":"1.8.3","minitest":"5.8.2","thread_safe":"0.3.5","tzinfo":"1.2.2","activesupport":"4.2.1","builder":"3.2.2","erubis":"2.7.0","mini_portile":"0.6.2","nokogiri":"1.6.6.2","rails-deprecated_sanitizer":"1.0.3","rails-dom-testing":"1.0.7","loofah":"2.0.3","rails-html-sanitizer":"1.0.2","actionview":"4.2.1","rack":"1.6.4","rack-test":"0.6.3","actionpack":"4.2.1","globalid":"0.3.6","activejob":"4.2.1","mime-types":"2.6.2","mail":"2.6.3","actionmailer":"4.2.1","activemodel":"4.2.1","arel":"6.0.3","activerecord":"4.2.1","debug_inspector":"0.0.2","binding_of_caller":"0.7.2","bundler":"1.11.2","coderay":"1.1.0","coffee-script-source":"1.10.0","execjs":"2.6.0","coffee-script":"2.4.1","thor":"0.19.1","railties":"4.2.1","coffee-rails":"4.1.0","multipart-post":"2.0.0","faraday":"0.9.2","multi_json":"1.11.2","jbuilder":"2.3.2","jquery-rails":"4.0.5","method_source":"0.8.2","slop":"3.6.0","pry":"0.10.3","sprockets":"3.4.0","sprockets-rails":"2.3.3","rails":"4.2.1","rdoc":"4.2.0","sass":"3.4.19","tilt":"2.0.1","sass-rails":"5.0.4","sdoc":"0.4.1","sentry-raven":"0.15.2","spring":"1.4.1","sqlite3":"1.3.11","turbolinks":"2.5.3","uglifier":"2.7.2","web-console":"2.2.1"},"extra":{},"tags":{},"errors":[{"type":"invalid_data","name":"timestamp","value":"2016-02-15T06:01:29"}],"interfaces":{"exception":{"values":[{"type":"ZeroDivisionError","value":"\"divided by 0\"","module":"","stacktrace":{"frames":[{"abs_path":"\/home\/sergiu\/.rvm\/rubies\/ruby-2.2.2\/lib\/ruby\/2.2.0\/webrick\/server.rb","filename":"webrick\/server.rb","function":"block in start_thread","context_line":" block ? block.call(sock) : run(sock)\n","pre_context":["module ActionController\n"," module ImplicitRender\n"," def send_action(method, *args)\n"],"post_context":[" default_render unless performed?\n"," ret\n"," end\n"],"lineno":4},{"abs_path":"\/home\/sergiu\/ravenapp\/app\/controllers\/home_controller.rb","filename":"app\/controllers\/home_controller.rb","function":"index","context_line":" 1\/0\n","pre_context":[" # Prevent CSRF attacks by raising an exception.\n"," # For APIs, you may want to use :null_session instead.\n"," def index\n"],"post_context":[" end\n","end\n",""],"lineno":5},{"abs_path":"\/home\/sergiu\/ravenapp\/app\/controllers\/home_controller.rb","filename":"app\/controllers\/home_controller.rb","function":"\/","context_line":" 1\/0\n","pre_context":[" # Prevent CSRF attacks by raising an exception.\n"," # For APIs, you may want to use :null_session instead.\n"," def index\n"],"post_context":[" end\n","end\n",""],"lineno":5}],"frames_omitted":null,"has_frames":true}}],"exc_omitted":null},"http":{"env":{"REMOTE_ADDR":"127.0.0.1","SERVER_NAME":"localhost","SERVER_PORT":"3001"},"headers":[{"host":"localhost:3001"},{"connection":"keep-alive"},{"accept":"text\/html,application\/xhtml+xml,application\/xml;q=0.9,image\/webp,*\/*;q=0.8"},{"upgrade_insecure_requests":"1"},{"user_agent":"Mozilla\/5.0 (X11; Linux x86_64) Gecko\/20100101 Firefox\/46.0"},{"accept_encoding":"gzip, deflate, sdch"},{"accept_language":"en-US,en;q=0.8"},{"cookie":"currentConfigName=%22default%22; pickedWebsite=1; _epiclogger_session=NTIwU2prYUd2T0dEd3FGQWE0WUFaL3RDY0huRGFnV1Z5TEhOQ3RtWUZZTTVRSzgvRTZvdEI2SFRsSVQ0ajVidHRWQnA5ck9wT3ZQU095N0dkWjEza0dpYWlmekxyRzFJbEFVNk5zRExmMzg3Q2c2MFBWdi9UYVUyWjdkWHVMNUFaWFJzZ2pEMGdwd3JJSGpSNjlEK2o3ZjlWZEhISEprK1hXOFNvaGdRdHg2MkFxN0lrcmlIdUtQazVWUjNGaWJvUGVYTHJncEc2OWhpaHBZbXNqcVhUcjM0ZWQ5bDFnWDBVSGlaOE5rdGxiOHNDU2NUS3BaSjd4eUZSRklzVnU5M3Z0TmJLUzF6ZWxjOGUrRmF2NkZ6ZCtGMUdoQVdFUSt0am9KT2lDODRMckJwbWQ1ZU5hV1hhZmt2bHdDZHZibEFmMExXNTI5Tmt..."},{"version":"HTTP\/1.1"}],"url":"http:\/\/localhost\/\/"}},"site":null,"environment":null,"version":"5"}' - issue2 = FactoryGirl.create(:issue, subscriber: subscriber, group: grouped_issue, data: mozilla_browser ) - - aggregations = grouped_issue.aggregations("browser") - expect(aggregations[0]["browser"]).to eq('Chrome') - expect(aggregations[1]["browser"]).to eq('Firefox') - end - - it 'returns different messages' do - different_message = "mesaju vietii" - issue3 = FactoryGirl.create(:issue, subscriber: subscriber, group: grouped_issue, message: different_message ) - - aggregations = grouped_issue.aggregations("message") - expect(aggregations[0]["message"]).to eq('ZeroDivisionError: divided by 0') - expect(aggregations[1]["message"]).to eq('mesaju vietii') - end - - it 'returns different subscribers' do - subscriber2 = FactoryGirl.create(:subscriber, website: website, email: 'gogu@yahoo.com') - issue4 = FactoryGirl.create(:issue, subscriber: subscriber2, group: grouped_issue) - - aggregations = grouped_issue.aggregations("subscriber") - expect(aggregations[0]["subscriber"]["name"]).to eq(subscriber.name) - expect(aggregations[1]["subscriber"]["name"]).to eq(subscriber2.name) - end - end end From 15b3494207e8a5ee94eafa578911ced6892cb8de Mon Sep 17 00:00:00 2001 From: panioglo Date: Mon, 22 Aug 2016 16:47:13 +0300 Subject: [PATCH 12/12] moved worker to the workers folder --- app/models/issue.rb | 8 -------- app/workers/aggregates_worker.rb | 7 +++++++ 2 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 app/workers/aggregates_worker.rb diff --git a/app/models/issue.rb b/app/models/issue.rb index 4e96edab..a5f6173b 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -146,12 +146,4 @@ def issue_created GroupedIssueMailer.error_occurred(self, member).deliver_later end end - - class AggregatesWorker - include Sidekiq::Worker - def perform(issue_id) - record = Issue.find(issue_id) - ErrorStore::Aggregates.new(record).handle_aggregates - end - end end diff --git a/app/workers/aggregates_worker.rb b/app/workers/aggregates_worker.rb new file mode 100644 index 00000000..236a82e9 --- /dev/null +++ b/app/workers/aggregates_worker.rb @@ -0,0 +1,7 @@ +class AggregatesWorker + include Sidekiq::Worker + def perform(issue_id) + record = Issue.find(issue_id) + ErrorStore::Aggregates.new(record).handle_aggregates + end +end \ No newline at end of file