From 2bc8c8b86cbf4bc8f6302e5de6b66a3b92aaca94 Mon Sep 17 00:00:00 2001 From: Davide Papagni Date: Thu, 24 Mar 2016 17:52:44 +0100 Subject: [PATCH 1/5] mongo document to hold user and checkins data to be imported --- backend/app/models/export.rb | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 backend/app/models/export.rb diff --git a/backend/app/models/export.rb b/backend/app/models/export.rb new file mode 100644 index 000000000..e0822c90b --- /dev/null +++ b/backend/app/models/export.rb @@ -0,0 +1,8 @@ +class Export + include Mongoid::Document + + field :user, type: Hash + field :profile, type: Hash + field :checkins, type: Array + +end From 0a7c147c683afb9bc4ca575c178f72b9e0388fe1 Mon Sep 17 00:00:00 2001 From: Davide Papagni Date: Thu, 24 Mar 2016 17:56:09 +0100 Subject: [PATCH 2/5] import rake tasks --- backend/lib/tasks/import.rake | 193 ++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 backend/lib/tasks/import.rake diff --git a/backend/lib/tasks/import.rake b/backend/lib/tasks/import.rake new file mode 100644 index 000000000..bace882ab --- /dev/null +++ b/backend/lib/tasks/import.rake @@ -0,0 +1,193 @@ +namespace :import do + + task from_json: :environment do + Export.delete_all + system('mongoimport -d flaredown_development -c exports --file exports.json --jsonArray') + end + + task users_and_checkins: :environment do + delete_users_and_checkins + Export.all.each do |export| + user = import_user(export.user) + puts " === #{user.email} ===" + import_profile(user, export.profile) + export.checkins.each { |checkin| import_checkin(user, checkin) } + set_trackings(user) + end + end + + def delete_users_and_checkins + print 'Deleting users and checkins... ' + Checkin::Condition.delete_all + Checkin::Symptom.delete_all + Checkin::Treatment.delete_all + Checkin.delete_all + User.destroy_all + puts 'Done' + end + + def import_user(source) + print "\nImporting user... " + user = User.new(source) + saved = user.save(validate: false) + fail ActiveRecord::RecordInvalid.new(user.errors.full_messages.join(',')) if not saved + puts 'Done' + user.reload + end + + def import_profile(user, source) + print ' > Importing profile... ' + user.profile.update_attributes!( + sex_id: import_sex(source[:sex]), + birth_date: import_birth_date(source[:dobYear], source[:dobMonth], source[:dobDay]), + country_id: import_country(source[:location]), + onboarding_step_id: import_onboarding_step(source[:onboarded]), + day_habit_id: import_day_habit(source[:occupation]), + ethnicity_ids: to_ethnicity_ids(source[:ethnicOrigin]), + education_level_id: import_education_level(source[:highestEducation]), + day_walking_hours: source[:activityLevel].to_i + ) + puts 'Done' + end + + def import_sex(source_sex) + return nil if source_sex.blank? + Sex.all.find { |s| s.name.upcase == source_sex.upcase }.id + end + + def import_birth_date(dob_year, dob_month, dob_day) + return nil if dob_year.blank? || dob_month.blank? || dob_day.blank? + Date.new(dob_year.to_i, dob_month.to_i, dob_day.to_i) + end + + def import_country(source_country) + return nil if source_country.blank? + wrong_countries_map = { + 'Macau SAR China' => 'Macao', + 'Hong Kong SAR China' => 'Hong Kong', + 'East Germany' => 'Germany' + } + source_country = wrong_countries_map[source_country] if wrong_countries_map.keys.include?(source_country) + country = Country.find_country_by_name(source_country) + if country.present? + country.alpha2 + else + print " FAILED: Country '#{source_country}' not found ... " + end + end + + def import_onboarding_step(onboarded) + (onboarded == 'true') ? 'onboarding-completed' : 'onboarding-personal' + end + + def import_day_habit(source_day_habit) + return nil if source_day_habit.blank? + DayHabit.all.find { |d| d.name.upcase == source_day_habit.upcase }.id + end + + def import_education_level(source_education_level) + return nil if source_education_level.blank? + EducationLevel.all.find { |e| e.name.upcase == source_education_level.upcase }.id + end + + def to_ethnicity_ids(ethnic_origins) + return [] if ethnic_origins.nil? || ethnic_origins == '[]' + YAML::load(ethnic_origins).map do |ethnic_origin| + Ethnicity.all.find { |e| e.name.upcase == ethnic_origin.upcase }.id + end + end + + def import_checkin(user, source) + date = source[:date] + pretty_date = date.strftime('%Y-%m-%d') + print " > Checkin: #{pretty_date}... " + begin + Checkin.create!( + user_id: user.id, + date: date, + note: source[:notes], + tag_ids: source[:tags].map { |name| Tag.find_or_create_by(name: name).id }, + conditions_attributes: import_conditions(user, source[:conditions]), + symptoms_attributes: import_symptoms(user, source[:symptoms]), + treatments_attributes: import_treatments(user, source[:treatments]) + ) + puts 'Done' + rescue Mongoid::Errors::Validations => e + puts "\n FAILED: #{e.summary} | Checkin: #{pretty_date}, User: #{user.email}" + end + end + + def import_conditions(user, sources) + sources.map do |source| + condition = Condition.new(name: source[:name], global: false) + condition = TrackableCreator.new(condition, user).create! + { + condition_id: condition.id, + value: source[:value] + } + end + end + + def import_symptoms(user, sources) + sources.map do |source| + symptom = Symptom.new(name: source[:name], global: false) + symptom = TrackableCreator.new(symptom, user).create! + { + symptom_id: symptom.id, + value: source[:value] + } + end + end + + def import_treatments(user, sources) + sources.map do |source| + treatment = Treatment.new(name: source[:name], global: false) + treatment = TrackableCreator.new(treatment, user).create! + { + treatment_id: treatment.id, + value: source[:value], + is_taken: source[:is_taken] + } + end + end + + def set_trackings(user) + first_checkin = user.checkins.sort(date: -1).first + return if first_checkin.nil? + start_at = first_checkin.date - 2.days + # Conditions + conditions = user.checkins.map(&:conditions).flatten + condition_ids = conditions.map(&:condition_id).uniq + condition_ids.each do |condition_id| + Tracking.create!( + start_at: start_at, + user: user, + trackable_type: 'Condition', + trackable_id: condition_id + ) + end + # Symptoms + symptoms = user.checkins.map(&:symptoms).flatten + symptom_ids = symptoms.map(&:symptom_id).uniq + symptom_ids.each do |symptom_id| + Tracking.create!( + start_at: start_at, + user: user, + trackable_type: 'Symptom', + trackable_id: symptom_id + ) + end + # Treatments + treatments = user.checkins.map(&:treatments).flatten + treatment_ids = treatments.map(&:treatment_id).uniq + treatment_ids.each do |treatment_id| + Tracking.create!( + start_at: start_at, + user: user, + trackable_type: 'Treatment', + trackable_id: treatment_id + ) + end + end + +end From c9c5be209054bcb3134ae3657ed6dc0906c77e64 Mon Sep 17 00:00:00 2001 From: Davide Papagni Date: Thu, 24 Mar 2016 22:54:12 +0100 Subject: [PATCH 3/5] import task: increase log level --- backend/lib/tasks/import.rake | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/lib/tasks/import.rake b/backend/lib/tasks/import.rake index bace882ab..24606714d 100644 --- a/backend/lib/tasks/import.rake +++ b/backend/lib/tasks/import.rake @@ -6,6 +6,7 @@ namespace :import do end task users_and_checkins: :environment do + Rails.logger.level = 3 delete_users_and_checkins Export.all.each do |export| user = import_user(export.user) From cf1cc5558d0c65995f0f0e18298b618597e8d76e Mon Sep 17 00:00:00 2001 From: Davide Papagni Date: Fri, 25 Mar 2016 16:21:14 +0100 Subject: [PATCH 4/5] import: from rake task to job, one per user --- backend/app/jobs/import_user_job.rb | 180 ++++++++++++++++++++++++++++ backend/config/sidekiq.yml | 3 +- backend/lib/tasks/import.rake | 175 +-------------------------- 3 files changed, 185 insertions(+), 173 deletions(-) create mode 100644 backend/app/jobs/import_user_job.rb diff --git a/backend/app/jobs/import_user_job.rb b/backend/app/jobs/import_user_job.rb new file mode 100644 index 000000000..aa91a5aa3 --- /dev/null +++ b/backend/app/jobs/import_user_job.rb @@ -0,0 +1,180 @@ +class ImportUserJob < ActiveJob::Base + queue_as :import + + def perform(export_id) + Rails.logger.level = 1 + export = Export.find(export_id) + user = import_user(export.user) + Rails.logger.info "=== #{user.email} ===" + import_profile(user, export.profile) + export.checkins.each { |checkin| import_checkin(user, checkin) } + set_trackings(user) + end + + def import_user(source) + Rails.logger.info "Importing user... " + user = User.new(source) + saved = user.save(validate: false) + fail ActiveRecord::RecordInvalid.new(user.errors.full_messages.join(',')) if not saved + Rails.logger.info 'Done' + user.reload + end + + def import_profile(user, source) + Rails.logger.info ' > Importing profile... ' + user.profile.update_attributes!( + sex_id: import_sex(source[:sex]), + birth_date: import_birth_date(source[:dobYear], source[:dobMonth], source[:dobDay]), + country_id: import_country(source[:location]), + onboarding_step_id: import_onboarding_step(source[:onboarded]), + day_habit_id: import_day_habit(source[:occupation]), + ethnicity_ids: to_ethnicity_ids(source[:ethnicOrigin]), + education_level_id: import_education_level(source[:highestEducation]), + day_walking_hours: source[:activityLevel].to_i + ) + Rails.logger.info ' Done' + end + + def import_sex(source_sex) + return nil if source_sex.blank? + Sex.all.find { |s| s.name.upcase == source_sex.upcase }.id + end + + def import_birth_date(dob_year, dob_month, dob_day) + return nil if dob_year.blank? || dob_month.blank? || dob_day.blank? + Date.new(dob_year.to_i, dob_month.to_i, dob_day.to_i) + end + + def import_country(source_country) + return nil if source_country.blank? + wrong_countries_map = { + 'Macau SAR China' => 'Macao', + 'Hong Kong SAR China' => 'Hong Kong', + 'East Germany' => 'Germany' + } + source_country = wrong_countries_map[source_country] if wrong_countries_map.keys.include?(source_country) + country = Country.find_country_by_name(source_country) + if country.present? + country.alpha2 + else + Rails.logger.info " FAILED: Country '#{source_country}' not found ... " + end + end + + def import_onboarding_step(onboarded) + (onboarded == 'true') ? 'onboarding-completed' : 'onboarding-personal' + end + + def import_day_habit(source_day_habit) + return nil if source_day_habit.blank? + DayHabit.all.find { |d| d.name.upcase == source_day_habit.upcase }.id + end + + def import_education_level(source_education_level) + return nil if source_education_level.blank? + EducationLevel.all.find { |e| e.name.upcase == source_education_level.upcase }.id + end + + def to_ethnicity_ids(ethnic_origins) + return [] if ethnic_origins.nil? || ethnic_origins == '[]' + YAML::load(ethnic_origins).map do |ethnic_origin| + Ethnicity.all.find { |e| e.name.upcase == ethnic_origin.upcase }.id + end + end + + def import_checkin(user, source) + date = source[:date] + pretty_date = date.strftime('%Y-%m-%d') + Rails.logger.info " > Checkin: #{pretty_date}... " + begin + Checkin.create!( + user_id: user.id, + date: date, + note: source[:notes], + tag_ids: source[:tags].map { |name| Tag.find_or_create_by(name: name).id }, + conditions_attributes: import_conditions(user, source[:conditions]), + symptoms_attributes: import_symptoms(user, source[:symptoms]), + treatments_attributes: import_treatments(user, source[:treatments]) + ) + Rails.logger.info ' Done' + rescue Mongoid::Errors::Validations => e + Rails.logger.info " FAILED: #{e.summary} | Checkin: #{pretty_date}, User: #{user.email}" + end + end + + def import_conditions(user, sources) + sources.map do |source| + condition = Condition.new(name: source[:name], global: false) + condition = TrackableCreator.new(condition, user).create! + { + condition_id: condition.id, + value: source[:value] + } + end + end + + def import_symptoms(user, sources) + sources.map do |source| + symptom = Symptom.new(name: source[:name], global: false) + symptom = TrackableCreator.new(symptom, user).create! + { + symptom_id: symptom.id, + value: source[:value] + } + end + end + + def import_treatments(user, sources) + sources.map do |source| + treatment = Treatment.new(name: source[:name], global: false) + treatment = TrackableCreator.new(treatment, user).create! + { + treatment_id: treatment.id, + value: source[:value], + is_taken: source[:is_taken] + } + end + end + + def set_trackings(user) + Rails.logger.info ' > Setting trackings... ' + first_checkin = user.checkins.sort(date: -1).first + return if first_checkin.nil? + start_at = first_checkin.date - 2.days + # Conditions + conditions = user.checkins.map(&:conditions).flatten + condition_ids = conditions.map(&:condition_id).uniq + condition_ids.each do |condition_id| + Tracking.create!( + start_at: start_at, + user: user, + trackable_type: 'Condition', + trackable_id: condition_id + ) + end + # Symptoms + symptoms = user.checkins.map(&:symptoms).flatten + symptom_ids = symptoms.map(&:symptom_id).uniq + symptom_ids.each do |symptom_id| + Tracking.create!( + start_at: start_at, + user: user, + trackable_type: 'Symptom', + trackable_id: symptom_id + ) + end + # Treatments + treatments = user.checkins.map(&:treatments).flatten + treatment_ids = treatments.map(&:treatment_id).uniq + treatment_ids.each do |treatment_id| + Tracking.create!( + start_at: start_at, + user: user, + trackable_type: 'Treatment', + trackable_id: treatment_id + ) + end + Rails.logger.info ' Done' + end + +end diff --git a/backend/config/sidekiq.yml b/backend/config/sidekiq.yml index 0079ffbd5..78d9ea1fb 100644 --- a/backend/config/sidekiq.yml +++ b/backend/config/sidekiq.yml @@ -1,6 +1,6 @@ --- :verbose: false -:concurrency: 2 +:concurrency: 1 # Set timeout to 8 on Heroku, longer if you manage your own systems. :timeout: 30 @@ -9,3 +9,4 @@ :queues: - <%= ENV['RACK_ENV'] %>.default - <%= ENV['RACK_ENV'] %>.mailers + - <%= ENV['RACK_ENV'] %>.import diff --git a/backend/lib/tasks/import.rake b/backend/lib/tasks/import.rake index 24606714d..f15a07e27 100644 --- a/backend/lib/tasks/import.rake +++ b/backend/lib/tasks/import.rake @@ -6,189 +6,20 @@ namespace :import do end task users_and_checkins: :environment do - Rails.logger.level = 3 delete_users_and_checkins Export.all.each do |export| - user = import_user(export.user) - puts " === #{user.email} ===" - import_profile(user, export.profile) - export.checkins.each { |checkin| import_checkin(user, checkin) } - set_trackings(user) + ImportUserJob.perform_later(export.id.to_s) end end def delete_users_and_checkins - print 'Deleting users and checkins... ' + Rails.logger.info 'Deleting users and checkins... ' Checkin::Condition.delete_all Checkin::Symptom.delete_all Checkin::Treatment.delete_all Checkin.delete_all User.destroy_all - puts 'Done' - end - - def import_user(source) - print "\nImporting user... " - user = User.new(source) - saved = user.save(validate: false) - fail ActiveRecord::RecordInvalid.new(user.errors.full_messages.join(',')) if not saved - puts 'Done' - user.reload - end - - def import_profile(user, source) - print ' > Importing profile... ' - user.profile.update_attributes!( - sex_id: import_sex(source[:sex]), - birth_date: import_birth_date(source[:dobYear], source[:dobMonth], source[:dobDay]), - country_id: import_country(source[:location]), - onboarding_step_id: import_onboarding_step(source[:onboarded]), - day_habit_id: import_day_habit(source[:occupation]), - ethnicity_ids: to_ethnicity_ids(source[:ethnicOrigin]), - education_level_id: import_education_level(source[:highestEducation]), - day_walking_hours: source[:activityLevel].to_i - ) - puts 'Done' - end - - def import_sex(source_sex) - return nil if source_sex.blank? - Sex.all.find { |s| s.name.upcase == source_sex.upcase }.id - end - - def import_birth_date(dob_year, dob_month, dob_day) - return nil if dob_year.blank? || dob_month.blank? || dob_day.blank? - Date.new(dob_year.to_i, dob_month.to_i, dob_day.to_i) - end - - def import_country(source_country) - return nil if source_country.blank? - wrong_countries_map = { - 'Macau SAR China' => 'Macao', - 'Hong Kong SAR China' => 'Hong Kong', - 'East Germany' => 'Germany' - } - source_country = wrong_countries_map[source_country] if wrong_countries_map.keys.include?(source_country) - country = Country.find_country_by_name(source_country) - if country.present? - country.alpha2 - else - print " FAILED: Country '#{source_country}' not found ... " - end - end - - def import_onboarding_step(onboarded) - (onboarded == 'true') ? 'onboarding-completed' : 'onboarding-personal' - end - - def import_day_habit(source_day_habit) - return nil if source_day_habit.blank? - DayHabit.all.find { |d| d.name.upcase == source_day_habit.upcase }.id - end - - def import_education_level(source_education_level) - return nil if source_education_level.blank? - EducationLevel.all.find { |e| e.name.upcase == source_education_level.upcase }.id - end - - def to_ethnicity_ids(ethnic_origins) - return [] if ethnic_origins.nil? || ethnic_origins == '[]' - YAML::load(ethnic_origins).map do |ethnic_origin| - Ethnicity.all.find { |e| e.name.upcase == ethnic_origin.upcase }.id - end - end - - def import_checkin(user, source) - date = source[:date] - pretty_date = date.strftime('%Y-%m-%d') - print " > Checkin: #{pretty_date}... " - begin - Checkin.create!( - user_id: user.id, - date: date, - note: source[:notes], - tag_ids: source[:tags].map { |name| Tag.find_or_create_by(name: name).id }, - conditions_attributes: import_conditions(user, source[:conditions]), - symptoms_attributes: import_symptoms(user, source[:symptoms]), - treatments_attributes: import_treatments(user, source[:treatments]) - ) - puts 'Done' - rescue Mongoid::Errors::Validations => e - puts "\n FAILED: #{e.summary} | Checkin: #{pretty_date}, User: #{user.email}" - end - end - - def import_conditions(user, sources) - sources.map do |source| - condition = Condition.new(name: source[:name], global: false) - condition = TrackableCreator.new(condition, user).create! - { - condition_id: condition.id, - value: source[:value] - } - end - end - - def import_symptoms(user, sources) - sources.map do |source| - symptom = Symptom.new(name: source[:name], global: false) - symptom = TrackableCreator.new(symptom, user).create! - { - symptom_id: symptom.id, - value: source[:value] - } - end - end - - def import_treatments(user, sources) - sources.map do |source| - treatment = Treatment.new(name: source[:name], global: false) - treatment = TrackableCreator.new(treatment, user).create! - { - treatment_id: treatment.id, - value: source[:value], - is_taken: source[:is_taken] - } - end - end - - def set_trackings(user) - first_checkin = user.checkins.sort(date: -1).first - return if first_checkin.nil? - start_at = first_checkin.date - 2.days - # Conditions - conditions = user.checkins.map(&:conditions).flatten - condition_ids = conditions.map(&:condition_id).uniq - condition_ids.each do |condition_id| - Tracking.create!( - start_at: start_at, - user: user, - trackable_type: 'Condition', - trackable_id: condition_id - ) - end - # Symptoms - symptoms = user.checkins.map(&:symptoms).flatten - symptom_ids = symptoms.map(&:symptom_id).uniq - symptom_ids.each do |symptom_id| - Tracking.create!( - start_at: start_at, - user: user, - trackable_type: 'Symptom', - trackable_id: symptom_id - ) - end - # Treatments - treatments = user.checkins.map(&:treatments).flatten - treatment_ids = treatments.map(&:treatment_id).uniq - treatment_ids.each do |treatment_id| - Tracking.create!( - start_at: start_at, - user: user, - trackable_type: 'Treatment', - trackable_id: treatment_id - ) - end + Rails.logger.info 'Done' end end From 29f60bbcec891851b378f06e8ff0ffd4b1add415 Mon Sep 17 00:00:00 2001 From: Davide Papagni Date: Fri, 25 Mar 2016 16:22:10 +0100 Subject: [PATCH 5/5] import: mount sidekiq Web UI for monitoring --- backend/Gemfile | 5 +++++ backend/Gemfile.lock | 8 ++++++++ backend/config/routes.rb | 3 +++ 3 files changed, 16 insertions(+) diff --git a/backend/Gemfile b/backend/Gemfile index 6a446162e..aa0c25fd2 100644 --- a/backend/Gemfile +++ b/backend/Gemfile @@ -31,6 +31,8 @@ gem 'colored', '~> 1.2' # Background jobs gem 'sidekiq', '~> 4.0' +# Needed by Sidekiq Web UI +gem 'sinatra', :require => nil # Structured seed data gem 'seedbank' @@ -44,6 +46,9 @@ gem 'pusher' # ActiveRecord data translations gem 'globalize', '~> 5.0.0' +# exception tracking +gem 'airbrake', '~> 5.0' + group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug' diff --git a/backend/Gemfile.lock b/backend/Gemfile.lock index d49560c05..d6d3cb40f 100644 --- a/backend/Gemfile.lock +++ b/backend/Gemfile.lock @@ -183,6 +183,8 @@ GEM quiet_assets (1.1.0) railties (>= 3.1, < 5.0) rack (1.6.4) + rack-protection (1.5.3) + rack rack-test (0.6.3) rack (>= 1.0) rails (4.2.5) @@ -258,6 +260,10 @@ GEM concurrent-ruby (~> 1.0) connection_pool (~> 2.2, >= 2.2.0) redis (~> 3.2, >= 3.2.1) + sinatra (1.4.7) + rack (~> 1.5) + rack-protection (~> 1.4) + tilt (>= 1.3, < 3) slop (3.6.0) spring (1.6.1) sprockets (3.5.2) @@ -269,6 +275,7 @@ GEM sprockets (>= 3.0.0) thor (0.19.1) thread_safe (0.3.5) + tilt (2.0.2) tins (1.6.0) tzinfo (1.2.2) thread_safe (~> 0.1) @@ -319,6 +326,7 @@ DEPENDENCIES seedbank shoulda-matchers sidekiq (~> 4.0) + sinatra spring tzinfo-data diff --git a/backend/config/routes.rb b/backend/config/routes.rb index f97fe1761..216b8b53f 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -9,6 +9,9 @@ omniauth_callbacks: 'api/v1/omniauth_callbacks' } + require 'sidekiq/web' + mount Sidekiq::Web => '/sidekiq' + # # API #