From c69b714747b8c3af845445f3ecd2601d7e9e48d0 Mon Sep 17 00:00:00 2001 From: Jason Cihelka Date: Tue, 6 Dec 2022 15:51:07 -0800 Subject: [PATCH 1/4] feat: add ShuttleSPPreference model --- model/shuttle.go | 1 + model/shuttle_preference.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 model/shuttle_preference.go diff --git a/model/shuttle.go b/model/shuttle.go index e37f55e5..549a58ab 100644 --- a/model/shuttle.go +++ b/model/shuttle.go @@ -16,4 +16,5 @@ type Shuttle struct { Private bool Open bool Priority int + PreferenceList ShuttleSPPreference } diff --git a/model/shuttle_preference.go b/model/shuttle_preference.go new file mode 100644 index 00000000..7fdb555a --- /dev/null +++ b/model/shuttle_preference.go @@ -0,0 +1,16 @@ +package model + +import "gorm.io/gorm" + +type SP struct { + gorm.Model + SPID string + Priority int + ShuttleSPPreferenceID uint +} + +type ShuttleSPPreference struct { + gorm.Model + StorageProviders []SP + ShuttleID string +} From ce2ea0aac56831020e3c6569d01ca5524674c199 Mon Sep 17 00:00:00 2001 From: Jason Cihelka Date: Wed, 7 Dec 2022 11:21:25 -0800 Subject: [PATCH 2/4] feat: rework shuttle_preference --- model/miner.go | 15 +++-- model/shuttle.go | 18 ++--- model/shuttle_preference.go | 14 ++-- sppreferencemanager/sppreferencemanager.go | 78 ++++++++++++++++++++++ 4 files changed, 99 insertions(+), 26 deletions(-) create mode 100644 sppreferencemanager/sppreferencemanager.go diff --git a/model/miner.go b/model/miner.go index 7749daf0..5e96ecde 100644 --- a/model/miner.go +++ b/model/miner.go @@ -7,11 +7,12 @@ import ( type StorageMiner struct { gorm.Model - Address util.DbAddr `gorm:"unique"` - Suspended bool - SuspendedReason string - Name string - Version string - Location string - Owner uint + Address util.DbAddr `gorm:"unique"` + Suspended bool + SuspendedReason string + Name string + Version string + Location string + Owner uint + ShuttlePreference []ShuttlePreference } diff --git a/model/shuttle.go b/model/shuttle.go index 549a58ab..97e51c73 100644 --- a/model/shuttle.go +++ b/model/shuttle.go @@ -8,13 +8,13 @@ import ( type Shuttle struct { gorm.Model - Handle string `gorm:"unique"` - Token string - Host string - PeerID string - LastConnection time.Time - Private bool - Open bool - Priority int - PreferenceList ShuttleSPPreference + Handle string `gorm:"unique"` + Token string + Host string + PeerID string + LastConnection time.Time + Private bool + Open bool + Priority int + ShuttlePreference []ShuttlePreference } diff --git a/model/shuttle_preference.go b/model/shuttle_preference.go index 7fdb555a..9a87243c 100644 --- a/model/shuttle_preference.go +++ b/model/shuttle_preference.go @@ -2,15 +2,9 @@ package model import "gorm.io/gorm" -type SP struct { +type ShuttlePreference struct { gorm.Model - SPID string - Priority int - ShuttleSPPreferenceID uint -} - -type ShuttleSPPreference struct { - gorm.Model - StorageProviders []SP - ShuttleID string + StorageMinerID uint + ShuttleID uint + Priority uint } diff --git a/sppreferencemanager/sppreferencemanager.go b/sppreferencemanager/sppreferencemanager.go new file mode 100644 index 00000000..cce23a79 --- /dev/null +++ b/sppreferencemanager/sppreferencemanager.go @@ -0,0 +1,78 @@ +package shuttlepreference + +import ( + "context" + "sort" + + "github.com/application-research/estuary/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type SPPreferenceManager struct { + DB *gorm.DB +} + +func NewSPPreferenceManager(db *gorm.DB) (*SPPreferenceManager, error) { + spref := &SPPreferenceManager{ + DB: db, + } + + return spref, nil +} + +// Get SPs and priority in reference to a given shuttle +func (spm *SPPreferenceManager) Query(sourceShuttle string) ([]model.ShuttlePreference, error) { + var spl []model.ShuttlePreference + + if err := spm.DB.Find(&spl, "ShuttleID = ?", sourceShuttle).Error; err != nil { + return nil, err + } + + return spl, nil +} + +type StorageProviderShuttlePing struct { + StorageMinerID uint + pings []ShuttlePing +} + +type ShuttlePing struct { + ShuttleID uint + ping float64 +} + +// Creates or update SP/Shuttle preference records associated with the provided SP ids +func (spm *SPPreferenceManager) PostSP(ctx context.Context, pings StorageProviderShuttlePing) error { + // Only create records for this max priority # + // TODO: Extract to config + maxPriorityToConsider := 2 + + var toCreate []model.ShuttlePreference + sorted := sortShuttles(pings.pings) + + // Create entries for each SP/Shuttle Priority mapping + for pri, shuttle := range sorted { + if pri == maxPriorityToConsider { + break + } + toCreate = append(toCreate, model.ShuttlePreference{ + StorageMinerID: pings.StorageMinerID, + ShuttleID: shuttle.ShuttleID, + Priority: uint(pri), + }) + } + + // Save entry - Upsert -> Update existing entries + spm.DB.Create(&toCreate).Clauses(clause.OnConflict{UpdateAll: true}) + return nil +} + +// Sorts a list of shuttles in increasing order of their ping +func sortShuttles(m []ShuttlePing) []ShuttlePing { + sort.Slice(m, func(i, j int) bool { + return m[i].ping < m[j].ping + }) + + return m +} From 6d8f644697c9e799970a36150ba4ef3a6d6f26a6 Mon Sep 17 00:00:00 2001 From: Jason Cihelka Date: Wed, 7 Dec 2022 11:29:56 -0800 Subject: [PATCH 3/4] feat: add index --- model/shuttle_preference.go | 4 ++-- sppreferencemanager/sppreferencemanager.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/model/shuttle_preference.go b/model/shuttle_preference.go index 9a87243c..2eb6bb6b 100644 --- a/model/shuttle_preference.go +++ b/model/shuttle_preference.go @@ -4,7 +4,7 @@ import "gorm.io/gorm" type ShuttlePreference struct { gorm.Model - StorageMinerID uint + StorageMinerID uint `gorm:"index:idx_pref"` ShuttleID uint - Priority uint + Priority uint `gorm:"index:idx_pref"` } diff --git a/sppreferencemanager/sppreferencemanager.go b/sppreferencemanager/sppreferencemanager.go index cce23a79..5d4445fd 100644 --- a/sppreferencemanager/sppreferencemanager.go +++ b/sppreferencemanager/sppreferencemanager.go @@ -63,7 +63,7 @@ func (spm *SPPreferenceManager) PostSP(ctx context.Context, pings StorageProvide }) } - // Save entry - Upsert -> Update existing entries + // Save entry - Upsert -> Update existing entries (if SP's shuttle priority changes) spm.DB.Create(&toCreate).Clauses(clause.OnConflict{UpdateAll: true}) return nil } From 3035af9412b27cae30fb3450fb5e5bab3c33e8f0 Mon Sep 17 00:00:00 2001 From: Jason Cihelka Date: Wed, 7 Dec 2022 12:08:15 -0800 Subject: [PATCH 4/4] feat: rename --- model/shuttle_preference.go | 2 +- .../shuttlepreferencemgr.go | 44 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) rename sppreferencemanager/sppreferencemanager.go => shuttlepreferencemgr/shuttlepreferencemgr.go (69%) diff --git a/model/shuttle_preference.go b/model/shuttle_preference.go index 2eb6bb6b..a2198800 100644 --- a/model/shuttle_preference.go +++ b/model/shuttle_preference.go @@ -6,5 +6,5 @@ type ShuttlePreference struct { gorm.Model StorageMinerID uint `gorm:"index:idx_pref"` ShuttleID uint - Priority uint `gorm:"index:idx_pref"` + Priority uint `gorm:"index:idx_pref, sort:asc"` } diff --git a/sppreferencemanager/sppreferencemanager.go b/shuttlepreferencemgr/shuttlepreferencemgr.go similarity index 69% rename from sppreferencemanager/sppreferencemanager.go rename to shuttlepreferencemgr/shuttlepreferencemgr.go index 5d4445fd..049912a1 100644 --- a/sppreferencemanager/sppreferencemanager.go +++ b/shuttlepreferencemgr/shuttlepreferencemgr.go @@ -1,4 +1,4 @@ -package shuttlepreference +package shuttlepreferencemgr import ( "context" @@ -9,20 +9,32 @@ import ( "gorm.io/gorm/clause" ) -type SPPreferenceManager struct { - DB *gorm.DB +type ShuttlePreferenceManager struct { + DB *gorm.DB + MaxPriority int } -func NewSPPreferenceManager(db *gorm.DB) (*SPPreferenceManager, error) { - spref := &SPPreferenceManager{ - DB: db, +type StorageProviderShuttlePing struct { + StorageMinerID uint + pings []ShuttlePing +} + +type ShuttlePing struct { + ShuttleID uint + ping float64 +} + +func NewShuttlePreferenceManager(db *gorm.DB, maxPriority int) (*ShuttlePreferenceManager, error) { + spref := &ShuttlePreferenceManager{ + DB: db, + MaxPriority: maxPriority, } return spref, nil } // Get SPs and priority in reference to a given shuttle -func (spm *SPPreferenceManager) Query(sourceShuttle string) ([]model.ShuttlePreference, error) { +func (spm *ShuttlePreferenceManager) Query(sourceShuttle string) ([]model.ShuttlePreference, error) { var spl []model.ShuttlePreference if err := spm.DB.Find(&spl, "ShuttleID = ?", sourceShuttle).Error; err != nil { @@ -32,28 +44,16 @@ func (spm *SPPreferenceManager) Query(sourceShuttle string) ([]model.ShuttlePref return spl, nil } -type StorageProviderShuttlePing struct { - StorageMinerID uint - pings []ShuttlePing -} - -type ShuttlePing struct { - ShuttleID uint - ping float64 -} - // Creates or update SP/Shuttle preference records associated with the provided SP ids -func (spm *SPPreferenceManager) PostSP(ctx context.Context, pings StorageProviderShuttlePing) error { - // Only create records for this max priority # - // TODO: Extract to config - maxPriorityToConsider := 2 +func (spm *ShuttlePreferenceManager) PostSP(ctx context.Context, pings StorageProviderShuttlePing) error { var toCreate []model.ShuttlePreference sorted := sortShuttles(pings.pings) // Create entries for each SP/Shuttle Priority mapping for pri, shuttle := range sorted { - if pri == maxPriorityToConsider { + // Only create records for this max priority # + if pri == spm.MaxPriority { break } toCreate = append(toCreate, model.ShuttlePreference{