From aea83082e7fe3067b8207e605b14fd985950b40d Mon Sep 17 00:00:00 2001 From: luc Date: Mon, 8 May 2023 13:06:28 -0300 Subject: [PATCH 01/25] added user storage threshold function --- util/users.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/util/users.go b/util/users.go index 7d1372c4..250ef8de 100644 --- a/util/users.go +++ b/util/users.go @@ -45,3 +45,19 @@ type InviteCode struct { CreatedBy uint ClaimedBy uint } + +// Max Storage per User 1.5 TB +const MaxUserStorageThreshold = int64(1_500_000_000_000) + +func IsUserReachedStorageThreshold(userId uint, db *gorm.DB) (bool, error) { + var sum int64 + err := db.Model(&Content{}). + Where("user_id = ?", userId). + Pluck("SUM(size)", &sum). + Error + + if err != nil { + return false, err + } + return sum >= MaxUserStorageThreshold, nil +} From 1c97dd50780ea38fc1e9b145f13475e42dbf58e5 Mon Sep 17 00:00:00 2001 From: luc Date: Mon, 8 May 2023 13:06:59 -0300 Subject: [PATCH 02/25] added bytes to TB utilility method --- util/misc.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/util/misc.go b/util/misc.go index e1af71b1..c7191696 100644 --- a/util/misc.go +++ b/util/misc.go @@ -209,3 +209,8 @@ func ToMultiAddress(addr string) (multiaddr.Multiaddr, error) { } return a, nil } + +func BytesToTB(bytes int64) float64 { + tb := float64(bytes) / (1024 * 1024 * 1024 * 1024) + return tb +} From ec15b85492661f2271240d956c962d13fcf57ca8 Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 9 May 2023 08:43:02 -0300 Subject: [PATCH 03/25] moving utilization query to own table --- util/users.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/util/users.go b/util/users.go index 250ef8de..7d1372c4 100644 --- a/util/users.go +++ b/util/users.go @@ -45,19 +45,3 @@ type InviteCode struct { CreatedBy uint ClaimedBy uint } - -// Max Storage per User 1.5 TB -const MaxUserStorageThreshold = int64(1_500_000_000_000) - -func IsUserReachedStorageThreshold(userId uint, db *gorm.DB) (bool, error) { - var sum int64 - err := db.Model(&Content{}). - Where("user_id = ?", userId). - Pluck("SUM(size)", &sum). - Error - - if err != nil { - return false, err - } - return sum >= MaxUserStorageThreshold, nil -} From 0a960e2a463435638e3693073983044b2d34b0d8 Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 9 May 2023 09:43:04 -0300 Subject: [PATCH 04/25] added users storage capacity struct --- util/users_storage_capacity.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 util/users_storage_capacity.go diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go new file mode 100644 index 00000000..0a34d77c --- /dev/null +++ b/util/users_storage_capacity.go @@ -0,0 +1,32 @@ +package util + +import ( + "gorm.io/gorm" +) + +type UsersStorageCapacity struct { + gorm.Model + + UserId uint + Size int64 + SoftLimit int64 + HardLimit int64 +} + +func (usc *UsersStorageCapacity) GetUserStorageCapacity(user *User, db *gorm.DB) error { + if err := db.Find(&usc, "user_id = ?", user.ID).Error; err != nil { + usc.Size = 0 + usc.UserId = user.ID + db.Save(&usc) + } + return nil +} + +func (usc *UsersStorageCapacity) Save(db *gorm.DB) { + db.Save(&usc) +} + +func (usc *UsersStorageCapacity) IncreaseAndValidateThreshold(add int64) bool { + usc.Size += add + return usc.Size <= usc.HardLimit +} From 2bdd650c5b542cd540dc38abb78259e2176a6838 Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 9 May 2023 09:45:27 -0300 Subject: [PATCH 05/25] added http error for reached threshold limit --- util/http.go | 1 + 1 file changed, 1 insertion(+) diff --git a/util/http.go b/util/http.go index 246832ad..74709e94 100644 --- a/util/http.go +++ b/util/http.go @@ -36,6 +36,7 @@ const ( ERR_CONTENT_ADDING_DISABLED = "ERR_CONTENT_ADDING_DISABLED" ERR_INVALID_INPUT = "ERR_INVALID_INPUT" ERR_CONTENT_SIZE_OVER_LIMIT = "ERR_CONTENT_SIZE_OVER_LIMIT" + ERR_USER_REACHED_STORAGE_TRESHOLD = "ERR_USER_REACHED_STORAGE_TRESHOLD" ERR_PEERING_PEERS_ADD_ERROR = "ERR_PEERING_PEERS_ADD_ERROR" ERR_PEERING_PEERS_REMOVE_ERROR = "ERR_PEERING_PEERS_REMOVE_ERROR" ERR_PEERING_PEERS_START_ERROR = "ERR_PEERING_PEERS_START_ERROR" From 6a8d8dff38aa613ace78f29bca058a4b5dc7b4a3 Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 9 May 2023 09:46:18 -0300 Subject: [PATCH 06/25] added limit theshold check in handleAdd --- api/v1/handlers.go | 12 ++++++++++++ util/users_storage_capacity.go | 4 ---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index 6b6c8c14..319b7586 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -539,6 +539,18 @@ func (s *apiV1) handleAdd(c echo.Context, u *util.User) error { return err } + var usc util.UsersStorageCapacity + usc.GetUserStorageCapacity(u, s.DB) + + // Increase and validate that the user storage threshold has not reached limit + if !usc.IncreaseAndValidateThreshold(mpf.Size) { + return &util.HttpError{ + Code: http.StatusBadRequest, + Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, + Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if you need additonal storage.", util.BytesToTB(usc.HardLimit)), + } + } + // if splitting is disabled and uploaded content size is greater than content size limit // reject the upload, as it will only get stuck and deals will never be made for it if !u.FlagSplitContent() && mpf.Size > s.cfg.Content.MaxSize { diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go index 0a34d77c..e60c4048 100644 --- a/util/users_storage_capacity.go +++ b/util/users_storage_capacity.go @@ -22,10 +22,6 @@ func (usc *UsersStorageCapacity) GetUserStorageCapacity(user *User, db *gorm.DB) return nil } -func (usc *UsersStorageCapacity) Save(db *gorm.DB) { - db.Save(&usc) -} - func (usc *UsersStorageCapacity) IncreaseAndValidateThreshold(add int64) bool { usc.Size += add return usc.Size <= usc.HardLimit From 54d8473c67ae46fe6f78f03b4e108c1a5f983e6e Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 9 May 2023 10:56:07 -0300 Subject: [PATCH 07/25] added default for soft / hard limit --- util/users_storage_capacity.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go index e60c4048..7fc93040 100644 --- a/util/users_storage_capacity.go +++ b/util/users_storage_capacity.go @@ -9,15 +9,15 @@ type UsersStorageCapacity struct { UserId uint Size int64 - SoftLimit int64 - HardLimit int64 + SoftLimit int64 `gorm:"default:1319413953331"` + HardLimit int64 `gorm:"default:1649267441664"` } func (usc *UsersStorageCapacity) GetUserStorageCapacity(user *User, db *gorm.DB) error { - if err := db.Find(&usc, "user_id = ?", user.ID).Error; err != nil { + if err := db.First(&usc, "user_id = ?", user.ID).Error; err != nil { usc.Size = 0 usc.UserId = user.ID - db.Save(&usc) + db.Create(&usc) } return nil } From 0facfdafd4c7f19449f823408f4831e7ad15ca43 Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 9 May 2023 14:09:37 -0300 Subject: [PATCH 08/25] added utility methods for calculating body size --- util/misc.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/util/misc.go b/util/misc.go index c7191696..1cf78251 100644 --- a/util/misc.go +++ b/util/misc.go @@ -1,6 +1,7 @@ package util import ( + "bytes" "context" "encoding/json" "fmt" @@ -19,6 +20,9 @@ import ( "github.com/multiformats/go-multihash" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + + "io" + "io/ioutil" ) func CanRestartTransfer(st *filclient.ChannelState) bool { @@ -214,3 +218,14 @@ func BytesToTB(bytes int64) float64 { tb := float64(bytes) / (1024 * 1024 * 1024 * 1024) return tb } + +func GetRequestBodySize(in io.ReadCloser) (int64, error) { + bdWriter := &bytes.Buffer{} + bdReader := io.TeeReader(in, bdWriter) + + bdSize, err := io.Copy(ioutil.Discard, bdReader) + if err != nil { + return 0, err + } + return bdSize, nil +} From 3a0b5299771b8f2f4fad70963d33f539c5d6db81 Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 9 May 2023 14:11:42 -0300 Subject: [PATCH 09/25] added capacity checks for /add-car, /add-ipfs and /create --- api/v1/handlers.go | 42 +++++++++++++++++++++++++++++++++++++++++- api/v1/pinning.go | 22 +++++++++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index 319b7586..d075698e 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -380,6 +380,23 @@ func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { return s.redirectContentAdding(c, u) } + bdSize, err := util.GetRequestBodySize(c.Request().Body) + if err != nil { + return err + } + + var usc util.UsersStorageCapacity + usc.GetUserStorageCapacity(u, s.DB) + + // Increase and validate that the user storage threshold has not reached limit + if !usc.IncreaseAndValidateThreshold(bdSize) { + return &util.HttpError{ + Code: http.StatusBadRequest, + Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, + Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed.", util.BytesToTB(usc.HardLimit)), + } + } + // if splitting is disabled and uploaded content size is greater than content size limit // reject the upload, as it will only get stuck and deals will never be made for it // if !u.FlagSplitContent() { @@ -470,6 +487,9 @@ func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { return err } + // Update user storage capacity with new value + s.DB.Save(&usc) + go func() { if err := s.nd.Provider.Provide(rootCID); err != nil { s.log.Warnf("failed to announce providers: %s", err) @@ -547,7 +567,7 @@ func (s *apiV1) handleAdd(c echo.Context, u *util.User) error { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, - Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if you need additonal storage.", util.BytesToTB(usc.HardLimit)), + Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed.", util.BytesToTB(usc.HardLimit)), } } @@ -4526,6 +4546,23 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { return err } + bdSize, err := util.GetRequestBodySize(c.Request().Body) + if err != nil { + return err + } + + var usc util.UsersStorageCapacity + usc.GetUserStorageCapacity(u, s.DB) + + // Increase and validate that the user storage threshold has not reached limit + if !usc.IncreaseAndValidateThreshold(bdSize) { + return &util.HttpError{ + Code: http.StatusBadRequest, + Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, + Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed.", util.BytesToTB(usc.HardLimit)), + } + } + rootCID, err := cid.Decode(req.Root) if err != nil { return err @@ -4590,6 +4627,9 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { } } + // Update user storage capacity with new value + s.DB.Save(&usc) + return c.JSON(http.StatusOK, util.ContentCreateResponse{ ID: content.ID, }) diff --git a/api/v1/pinning.go b/api/v1/pinning.go index 4d58ea7c..3c845c9e 100644 --- a/api/v1/pinning.go +++ b/api/v1/pinning.go @@ -262,6 +262,23 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { overwrite = true } + bdSize, err := util.GetRequestBodySize(e.Request().Body) + if err != nil { + return err + } + + var usc util.UsersStorageCapacity + usc.GetUserStorageCapacity(u, s.DB) + + // Increase and validate that the user storage threshold has not reached limit + if !usc.IncreaseAndValidateThreshold(bdSize) { + return &util.HttpError{ + Code: http.StatusBadRequest, + Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, + Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed.", util.BytesToTB(usc.HardLimit)), + } + } + ignoreDuplicates := false if c.QueryParam("ignore-dupes") == "true" { ignoreDuplicates = true @@ -281,7 +298,10 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { return err } - return c.JSON(http.StatusAccepted, status) + // Update user storage capacity with new value + s.DB.Save(&usc) + + return e.JSON(http.StatusAccepted, status) } // handleGetPin godoc From 8231f11462346cc0177e3475b57b1bb27af39365 Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 9 May 2023 14:12:10 -0300 Subject: [PATCH 10/25] added comments for default values --- util/users_storage_capacity.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go index 7fc93040..0523be64 100644 --- a/util/users_storage_capacity.go +++ b/util/users_storage_capacity.go @@ -9,8 +9,8 @@ type UsersStorageCapacity struct { UserId uint Size int64 - SoftLimit int64 `gorm:"default:1319413953331"` - HardLimit int64 `gorm:"default:1649267441664"` + SoftLimit int64 `gorm:"default:1319413953331"` // Hardlimit*.8 + HardLimit int64 `gorm:"default:1649267441664"` // 1.5TB } func (usc *UsersStorageCapacity) GetUserStorageCapacity(user *User, db *gorm.DB) error { @@ -19,6 +19,7 @@ func (usc *UsersStorageCapacity) GetUserStorageCapacity(user *User, db *gorm.DB) usc.UserId = user.ID db.Create(&usc) } + // check if cache is outdated and resync? return nil } From 278a2eac9cbc4cf8067e7c8d9b6c25153fb59453 Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 10 May 2023 10:25:13 -0300 Subject: [PATCH 11/25] introduced lastSync for forcing size refresh after 24h --- tests/data/files/large-file | 10 --------- util/users_storage_capacity.go | 38 ++++++++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) delete mode 100644 tests/data/files/large-file diff --git a/tests/data/files/large-file b/tests/data/files/large-file deleted file mode 100644 index cbab840c..00000000 --- a/tests/data/files/large-file +++ /dev/null @@ -1,10 +0,0 @@ -this is for a large file -this is for a large file -this is for a large file -this is for a large file -this is for a large file -this is for a large file -this is for a large file -this is for a large file -this is for a large file -this is for a large file diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go index 0523be64..61a78cba 100644 --- a/util/users_storage_capacity.go +++ b/util/users_storage_capacity.go @@ -2,24 +2,41 @@ package util import ( "gorm.io/gorm" + "time" ) type UsersStorageCapacity struct { gorm.Model - UserId uint - Size int64 - SoftLimit int64 `gorm:"default:1319413953331"` // Hardlimit*.8 - HardLimit int64 `gorm:"default:1649267441664"` // 1.5TB + UserId uint + Size int64 `gorm:"default:0"` + SoftLimit int64 `gorm:"default:1319413953331"` // Hardlimit*.8 + HardLimit int64 `gorm:"default:1649267441664"` // 1.5TB + LastSyncAt time.Time } +type Utilization struct { + TotalSize int64 +} + +// CutOverUtilizationDate All content uploaded pass this date will count toward your user storage capacity +const CutOverUtilizationDate = "2023-05-12 00:00:00" + func (usc *UsersStorageCapacity) GetUserStorageCapacity(user *User, db *gorm.DB) error { if err := db.First(&usc, "user_id = ?", user.ID).Error; err != nil { - usc.Size = 0 usc.UserId = user.ID db.Create(&usc) } - // check if cache is outdated and resync? + if isSyncNeeded(usc.LastSyncAt) { + var usage Utilization + if err := db.Raw(`SELECT (SELECT SUM(size) FROM contents where user_id = ? AND created_at >= ? AND NOT aggregate AND active AND deleted_at IS NULL) as total_size`, user.ID, CutOverUtilizationDate). + Scan(&usage).Error; err != nil { + return err + } + usc.Size = usage.TotalSize + usc.LastSyncAt = time.Now() + db.Save(&usc) + } return nil } @@ -27,3 +44,12 @@ func (usc *UsersStorageCapacity) IncreaseAndValidateThreshold(add int64) bool { usc.Size += add return usc.Size <= usc.HardLimit } + +const SyncRefreshInHours = 24 + +func isSyncNeeded(t time.Time) bool { + now := time.Now().UTC() + duration := now.Sub(t.UTC()) + refresh := SyncRefreshInHours * time.Hour + return duration >= refresh +} From e494d200525a156dbe722bead9149442592c03d0 Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 10 May 2023 13:05:42 -0300 Subject: [PATCH 12/25] added rest api endpoint for fetching user utilization stats --- api/v1/api.go | 1 + api/v1/handlers.go | 17 +++++++++++++++++ util/users_storage_capacity.go | 10 +++++----- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/api/v1/api.go b/api/v1/api.go index 852e1e51..0e8f5208 100644 --- a/api/v1/api.go +++ b/api/v1/api.go @@ -126,6 +126,7 @@ func (s *apiV1) RegisterRoutes(e *echo.Echo) { user.PUT("/password", util.WithUser(s.handleUserChangePassword)) user.PUT("/address", util.WithUser(s.handleUserChangeAddress)) user.GET("/stats", util.WithUser(s.handleGetUserStats)) + user.GET("/utilization", util.WithUser(s.handleGetUserUtilization)) userMiner := user.Group("/miner") userMiner.POST("/claim", util.WithUser(s.handleUserClaimMiner)) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index d075698e..5f96983f 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -2884,6 +2884,23 @@ func (s *apiV1) handleGetUserStats(c echo.Context, u *util.User) error { return c.JSON(http.StatusOK, stats) } +// handleGetUserStats godoc +// @Summary Gets User Utilization Stats +// @Description This endpoint is used to get utilization stats for the current user. +// @Tags User +// @Produce json +// @Success 200 {object} string +// @Failure 400 {object} util.HttpError +// @Failure 500 {object} util.HttpError +// @Router /user/utilization [get] +func (s *apiV1) handleGetUserUtilization(c echo.Context, u *util.User) error { + var usc util.UsersStorageCapacity + if err := usc.GetUserStorageCapacity(u, s.DB); err != nil { + return err + } + return c.JSON(http.StatusOK, usc) +} + func (s *apiV1) newAuthTokenForUser(user *util.User, expiry time.Time, perms []string, label string, isSession bool) (*util.AuthToken, error) { if len(perms) > 1 { return nil, fmt.Errorf("invalid perms") diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go index 61a78cba..9d010583 100644 --- a/util/users_storage_capacity.go +++ b/util/users_storage_capacity.go @@ -8,11 +8,11 @@ import ( type UsersStorageCapacity struct { gorm.Model - UserId uint - Size int64 `gorm:"default:0"` - SoftLimit int64 `gorm:"default:1319413953331"` // Hardlimit*.8 - HardLimit int64 `gorm:"default:1649267441664"` // 1.5TB - LastSyncAt time.Time + UserId uint `json:"user_id"` + Size int64 `json:"size" gorm:"default:0"` + SoftLimit int64 `json:"soft_limit" gorm:"default:1319413953331"` // Hardlimit*.8 + HardLimit int64 `json:"hard_limit" gorm:"default:1649267441664"` // 1.5TB + LastSyncAt time.Time `json:"last_sync_at"` } type Utilization struct { From 6dff853a9fcd5d21167b34c344d9f9bc8ea27902 Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 10 May 2023 14:28:27 -0300 Subject: [PATCH 13/25] fixed merge conficts --- tests/data/files/large-file | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/data/files/large-file diff --git a/tests/data/files/large-file b/tests/data/files/large-file new file mode 100644 index 00000000..cbab840c --- /dev/null +++ b/tests/data/files/large-file @@ -0,0 +1,10 @@ +this is for a large file +this is for a large file +this is for a large file +this is for a large file +this is for a large file +this is for a large file +this is for a large file +this is for a large file +this is for a large file +this is for a large file From 1785d6a92c3fc4039e6c95d617a41958762c189f Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 10 May 2023 14:42:24 -0300 Subject: [PATCH 14/25] fixed merge conficts --- api/v1/handlers.go | 12 ++++++------ api/v1/pinning.go | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index 5f96983f..2678a5e3 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -386,7 +386,7 @@ func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { } var usc util.UsersStorageCapacity - usc.GetUserStorageCapacity(u, s.DB) + usc.GetUserStorageCapacity(u, s.db) // Increase and validate that the user storage threshold has not reached limit if !usc.IncreaseAndValidateThreshold(bdSize) { @@ -488,7 +488,7 @@ func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { } // Update user storage capacity with new value - s.DB.Save(&usc) + s.db.Save(&usc) go func() { if err := s.nd.Provider.Provide(rootCID); err != nil { @@ -560,7 +560,7 @@ func (s *apiV1) handleAdd(c echo.Context, u *util.User) error { } var usc util.UsersStorageCapacity - usc.GetUserStorageCapacity(u, s.DB) + usc.GetUserStorageCapacity(u, s.db) // Increase and validate that the user storage threshold has not reached limit if !usc.IncreaseAndValidateThreshold(mpf.Size) { @@ -2895,7 +2895,7 @@ func (s *apiV1) handleGetUserStats(c echo.Context, u *util.User) error { // @Router /user/utilization [get] func (s *apiV1) handleGetUserUtilization(c echo.Context, u *util.User) error { var usc util.UsersStorageCapacity - if err := usc.GetUserStorageCapacity(u, s.DB); err != nil { + if err := usc.GetUserStorageCapacity(u, s.db); err != nil { return err } return c.JSON(http.StatusOK, usc) @@ -4569,7 +4569,7 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { } var usc util.UsersStorageCapacity - usc.GetUserStorageCapacity(u, s.DB) + usc.GetUserStorageCapacity(u, s.db) // Increase and validate that the user storage threshold has not reached limit if !usc.IncreaseAndValidateThreshold(bdSize) { @@ -4645,7 +4645,7 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { } // Update user storage capacity with new value - s.DB.Save(&usc) + s.db.Save(&usc) return c.JSON(http.StatusOK, util.ContentCreateResponse{ ID: content.ID, diff --git a/api/v1/pinning.go b/api/v1/pinning.go index 3c845c9e..3ce8594a 100644 --- a/api/v1/pinning.go +++ b/api/v1/pinning.go @@ -262,13 +262,13 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { overwrite = true } - bdSize, err := util.GetRequestBodySize(e.Request().Body) + bdSize, err := util.GetRequestBodySize(c.Request().Body) if err != nil { return err } var usc util.UsersStorageCapacity - usc.GetUserStorageCapacity(u, s.DB) + usc.GetUserStorageCapacity(u, s.db) // Increase and validate that the user storage threshold has not reached limit if !usc.IncreaseAndValidateThreshold(bdSize) { @@ -299,9 +299,9 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { } // Update user storage capacity with new value - s.DB.Save(&usc) + s.db.Save(&usc) - return e.JSON(http.StatusAccepted, status) + return c.JSON(http.StatusAccepted, status) } // handleGetPin godoc From 42a3d0ed95ed491bd2c9a35b222c7ac6640f01d3 Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 10 May 2023 14:58:22 -0300 Subject: [PATCH 15/25] added better error handling for gosec --- api/v1/handlers.go | 12 +++++++++--- api/v1/pinning.go | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index 2678a5e3..4ec083c0 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -386,7 +386,9 @@ func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { } var usc util.UsersStorageCapacity - usc.GetUserStorageCapacity(u, s.db) + if err := usc.GetUserStorageCapacity(u, s.db); err != nil { + return err + } // Increase and validate that the user storage threshold has not reached limit if !usc.IncreaseAndValidateThreshold(bdSize) { @@ -560,7 +562,9 @@ func (s *apiV1) handleAdd(c echo.Context, u *util.User) error { } var usc util.UsersStorageCapacity - usc.GetUserStorageCapacity(u, s.db) + if err := usc.GetUserStorageCapacity(u, s.db); err != nil { + return err + } // Increase and validate that the user storage threshold has not reached limit if !usc.IncreaseAndValidateThreshold(mpf.Size) { @@ -4569,7 +4573,9 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { } var usc util.UsersStorageCapacity - usc.GetUserStorageCapacity(u, s.db) + if err := usc.GetUserStorageCapacity(u, s.db); err != nil { + return err + } // Increase and validate that the user storage threshold has not reached limit if !usc.IncreaseAndValidateThreshold(bdSize) { diff --git a/api/v1/pinning.go b/api/v1/pinning.go index 3ce8594a..2abc5390 100644 --- a/api/v1/pinning.go +++ b/api/v1/pinning.go @@ -268,7 +268,9 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { } var usc util.UsersStorageCapacity - usc.GetUserStorageCapacity(u, s.db) + if err := usc.GetUserStorageCapacity(u, s.db); err != nil { + return err + } // Increase and validate that the user storage threshold has not reached limit if !usc.IncreaseAndValidateThreshold(bdSize) { From 08b9bad61499e13a8f3e907054fb6293dd65d655 Mon Sep 17 00:00:00 2001 From: luc Date: Thu, 11 May 2023 09:27:13 -0300 Subject: [PATCH 16/25] change io.reader to use content length for calculating request size --- api/v1/handlers.go | 16 +++++----------- api/v1/pinning.go | 9 +++------ util/misc.go | 15 --------------- 3 files changed, 8 insertions(+), 32 deletions(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index 4ec083c0..77c6d999 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -372,6 +372,8 @@ func (s *apiV1) handleAddIpfs(c echo.Context, u *util.User) error { func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { ctx := c.Request().Context() + contentLength := c.Request().ContentLength + if err := util.ErrorIfContentAddingDisabled(s.isContentAddingDisabled(u)); err != nil { return err } @@ -380,18 +382,13 @@ func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { return s.redirectContentAdding(c, u) } - bdSize, err := util.GetRequestBodySize(c.Request().Body) - if err != nil { - return err - } - var usc util.UsersStorageCapacity if err := usc.GetUserStorageCapacity(u, s.db); err != nil { return err } // Increase and validate that the user storage threshold has not reached limit - if !usc.IncreaseAndValidateThreshold(bdSize) { + if !usc.IncreaseAndValidateThreshold(contentLength) { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, @@ -4567,10 +4564,7 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { return err } - bdSize, err := util.GetRequestBodySize(c.Request().Body) - if err != nil { - return err - } + contentLength := c.Request().ContentLength var usc util.UsersStorageCapacity if err := usc.GetUserStorageCapacity(u, s.db); err != nil { @@ -4578,7 +4572,7 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { } // Increase and validate that the user storage threshold has not reached limit - if !usc.IncreaseAndValidateThreshold(bdSize) { + if !usc.IncreaseAndValidateThreshold(contentLength) { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, diff --git a/api/v1/pinning.go b/api/v1/pinning.go index 2abc5390..07f941ba 100644 --- a/api/v1/pinning.go +++ b/api/v1/pinning.go @@ -253,6 +253,8 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { return err } + contentLength := c.Request().ContentLength + if err := util.ErrorIfContentAddingDisabled(s.isContentAddingDisabled(u)); err != nil { return err } @@ -262,18 +264,13 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { overwrite = true } - bdSize, err := util.GetRequestBodySize(c.Request().Body) - if err != nil { - return err - } - var usc util.UsersStorageCapacity if err := usc.GetUserStorageCapacity(u, s.db); err != nil { return err } // Increase and validate that the user storage threshold has not reached limit - if !usc.IncreaseAndValidateThreshold(bdSize) { + if !usc.IncreaseAndValidateThreshold(contentLength) { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, diff --git a/util/misc.go b/util/misc.go index 1cf78251..c7191696 100644 --- a/util/misc.go +++ b/util/misc.go @@ -1,7 +1,6 @@ package util import ( - "bytes" "context" "encoding/json" "fmt" @@ -20,9 +19,6 @@ import ( "github.com/multiformats/go-multihash" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" - - "io" - "io/ioutil" ) func CanRestartTransfer(st *filclient.ChannelState) bool { @@ -218,14 +214,3 @@ func BytesToTB(bytes int64) float64 { tb := float64(bytes) / (1024 * 1024 * 1024 * 1024) return tb } - -func GetRequestBodySize(in io.ReadCloser) (int64, error) { - bdWriter := &bytes.Buffer{} - bdReader := io.TeeReader(in, bdWriter) - - bdSize, err := io.Copy(ioutil.Discard, bdReader) - if err != nil { - return 0, err - } - return bdSize, nil -} From 1a87d30e341a232afd9e32c7a01d25e9b6fe3cbb Mon Sep 17 00:00:00 2001 From: luc Date: Thu, 11 May 2023 11:51:56 -0300 Subject: [PATCH 17/25] added validate only function for pinning op --- api/v1/handlers.go | 7 +------ api/v1/pinning.go | 7 +------ util/users_storage_capacity.go | 4 ++++ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index 77c6d999..40457f0a 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -4564,15 +4564,13 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { return err } - contentLength := c.Request().ContentLength - var usc util.UsersStorageCapacity if err := usc.GetUserStorageCapacity(u, s.db); err != nil { return err } // Increase and validate that the user storage threshold has not reached limit - if !usc.IncreaseAndValidateThreshold(contentLength) { + if !usc.ValidateThreshold() { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, @@ -4644,9 +4642,6 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { } } - // Update user storage capacity with new value - s.db.Save(&usc) - return c.JSON(http.StatusOK, util.ContentCreateResponse{ ID: content.ID, }) diff --git a/api/v1/pinning.go b/api/v1/pinning.go index 07f941ba..2433a456 100644 --- a/api/v1/pinning.go +++ b/api/v1/pinning.go @@ -253,8 +253,6 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { return err } - contentLength := c.Request().ContentLength - if err := util.ErrorIfContentAddingDisabled(s.isContentAddingDisabled(u)); err != nil { return err } @@ -270,7 +268,7 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { } // Increase and validate that the user storage threshold has not reached limit - if !usc.IncreaseAndValidateThreshold(contentLength) { + if !usc.ValidateThreshold() { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, @@ -297,9 +295,6 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { return err } - // Update user storage capacity with new value - s.db.Save(&usc) - return c.JSON(http.StatusAccepted, status) } diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go index 9d010583..a93be687 100644 --- a/util/users_storage_capacity.go +++ b/util/users_storage_capacity.go @@ -45,6 +45,10 @@ func (usc *UsersStorageCapacity) IncreaseAndValidateThreshold(add int64) bool { return usc.Size <= usc.HardLimit } +func (usc *UsersStorageCapacity) ValidateThreshold() bool { + return usc.Size <= usc.HardLimit +} + const SyncRefreshInHours = 24 func isSyncNeeded(t time.Time) bool { From aefa16645f496325a7e3b488f2c0c63b1dfa06a7 Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 17 May 2023 09:17:37 -0300 Subject: [PATCH 18/25] cleanup storage threshold logic --- api/v1/handlers.go | 55 +++++++++++++++++++++------------- api/v1/pinning.go | 6 ++-- util/users_storage_capacity.go | 30 ++----------------- 3 files changed, 41 insertions(+), 50 deletions(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index 40457f0a..ae15ab05 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -372,7 +372,7 @@ func (s *apiV1) handleAddIpfs(c echo.Context, u *util.User) error { func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { ctx := c.Request().Context() - contentLength := c.Request().ContentLength + //contentLength := c.Request().ContentLength if err := util.ErrorIfContentAddingDisabled(s.isContentAddingDisabled(u)); err != nil { return err @@ -382,13 +382,13 @@ func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { return s.redirectContentAdding(c, u) } - var usc util.UsersStorageCapacity - if err := usc.GetUserStorageCapacity(u, s.db); err != nil { + // Get user storage capacity + usc, err := s.getUserStorageCapacity(u) + if err != nil { return err } - // Increase and validate that the user storage threshold has not reached limit - if !usc.IncreaseAndValidateThreshold(contentLength) { + if !usc.ValidateThreshold() { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, @@ -486,9 +486,6 @@ func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { return err } - // Update user storage capacity with new value - s.db.Save(&usc) - go func() { if err := s.nd.Provider.Provide(rootCID); err != nil { s.log.Warnf("failed to announce providers: %s", err) @@ -511,6 +508,24 @@ func (s *apiV1) loadCar(ctx context.Context, bs blockstore.Blockstore, r io.Read return car.LoadCar(ctx, bs, r) } +func (s *apiV1) getUserStorageCapacity(user *util.User) (*util.UsersStorageCapacity, error) { + var usc *util.UsersStorageCapacity + err := s.db.First(&usc, "user_id = ?", user.ID).Error + + if err != nil || usc.IsSyncNeeded() { + var usage util.Utilization + if err := s.db.Raw(`SELECT (SELECT SUM(size) FROM contents where user_id = ? AND created_at >= ? AND NOT aggregate AND active AND deleted_at IS NULL) as total_size`, user.ID, util.CutOverUtilizationDate). + Scan(&usage).Error; err != nil { + return usc, err + } + usc.Size = usage.TotalSize + usc.LastSyncAt = time.Now() + s.db.Save(&usc) + } + + return usc, err +} + // handleAdd godoc // @Summary Add new content // @Description This endpoint is used to upload new content. @@ -558,13 +573,13 @@ func (s *apiV1) handleAdd(c echo.Context, u *util.User) error { return err } - var usc util.UsersStorageCapacity - if err := usc.GetUserStorageCapacity(u, s.db); err != nil { + // Get user storage capacity + usc, err := s.getUserStorageCapacity(u) + if err != nil { return err } - // Increase and validate that the user storage threshold has not reached limit - if !usc.IncreaseAndValidateThreshold(mpf.Size) { + if !usc.ValidateThreshold() { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, @@ -891,9 +906,9 @@ func (s *apiV1) handleGetContent(c echo.Context, u *util.User) error { return err } - if err := util.IsContentOwner(u.ID, content.UserID); err != nil { - return err - } + //if err := util.IsContentOwner(u.ID, content.UserID); err != nil { + // return err + //} return c.JSON(http.StatusOK, content) } @@ -2895,8 +2910,8 @@ func (s *apiV1) handleGetUserStats(c echo.Context, u *util.User) error { // @Failure 500 {object} util.HttpError // @Router /user/utilization [get] func (s *apiV1) handleGetUserUtilization(c echo.Context, u *util.User) error { - var usc util.UsersStorageCapacity - if err := usc.GetUserStorageCapacity(u, s.db); err != nil { + usc, err := s.getUserStorageCapacity(u) + if err != nil { return err } return c.JSON(http.StatusOK, usc) @@ -4564,12 +4579,12 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { return err } - var usc util.UsersStorageCapacity - if err := usc.GetUserStorageCapacity(u, s.db); err != nil { + // Get user storage capacity + usc, err := s.getUserStorageCapacity(u) + if err != nil { return err } - // Increase and validate that the user storage threshold has not reached limit if !usc.ValidateThreshold() { return &util.HttpError{ Code: http.StatusBadRequest, diff --git a/api/v1/pinning.go b/api/v1/pinning.go index 2433a456..1107635c 100644 --- a/api/v1/pinning.go +++ b/api/v1/pinning.go @@ -262,12 +262,12 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { overwrite = true } - var usc util.UsersStorageCapacity - if err := usc.GetUserStorageCapacity(u, s.db); err != nil { + // Get user storage capacity + usc, err := s.getUserStorageCapacity(u) + if err != nil { return err } - // Increase and validate that the user storage threshold has not reached limit if !usc.ValidateThreshold() { return &util.HttpError{ Code: http.StatusBadRequest, diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go index a93be687..f694ee86 100644 --- a/util/users_storage_capacity.go +++ b/util/users_storage_capacity.go @@ -21,39 +21,15 @@ type Utilization struct { // CutOverUtilizationDate All content uploaded pass this date will count toward your user storage capacity const CutOverUtilizationDate = "2023-05-12 00:00:00" - -func (usc *UsersStorageCapacity) GetUserStorageCapacity(user *User, db *gorm.DB) error { - if err := db.First(&usc, "user_id = ?", user.ID).Error; err != nil { - usc.UserId = user.ID - db.Create(&usc) - } - if isSyncNeeded(usc.LastSyncAt) { - var usage Utilization - if err := db.Raw(`SELECT (SELECT SUM(size) FROM contents where user_id = ? AND created_at >= ? AND NOT aggregate AND active AND deleted_at IS NULL) as total_size`, user.ID, CutOverUtilizationDate). - Scan(&usage).Error; err != nil { - return err - } - usc.Size = usage.TotalSize - usc.LastSyncAt = time.Now() - db.Save(&usc) - } - return nil -} - -func (usc *UsersStorageCapacity) IncreaseAndValidateThreshold(add int64) bool { - usc.Size += add - return usc.Size <= usc.HardLimit -} +const SyncRefreshInHours = 24 func (usc *UsersStorageCapacity) ValidateThreshold() bool { return usc.Size <= usc.HardLimit } -const SyncRefreshInHours = 24 - -func isSyncNeeded(t time.Time) bool { +func (usc *UsersStorageCapacity) IsSyncNeeded() bool { now := time.Now().UTC() - duration := now.Sub(t.UTC()) + duration := now.Sub(usc.LastSyncAt.UTC()) refresh := SyncRefreshInHours * time.Hour return duration >= refresh } From 21180d2a5e729055cbd99c1aaedd24e00cf3648f Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 17 May 2023 09:19:34 -0300 Subject: [PATCH 19/25] change user size calculation to addObjectsToDatabase() once pinned --- pinner/block/block.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pinner/block/block.go b/pinner/block/block.go index aa5b7418..a95b79dc 100644 --- a/pinner/block/block.go +++ b/pinner/block/block.go @@ -96,6 +96,16 @@ func (m *manager) addObjectsToDatabase(ctx context.Context, cont *util.Content, return xerrors.Errorf("failed to update content in database: %w", err) } + var usc *util.UsersStorageCapacity + err := tx.First(&usc, "user_id = ?", cont.UserID).Error + if err != nil { + usc.UserId = cont.UserID + usc.Size = 0 + } + + usc.Size += contSize + tx.Save(&usc) + // if content can be staged, stage it if contSize < m.cfg.Content.MinSize { return m.stgZoneQueueMgr.QueueContent(cont, tx, false) From 4de9f2051d5d125c6c154150f773e8be320ee6fa Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 17 May 2023 09:32:25 -0300 Subject: [PATCH 20/25] cleanup comments --- api/v1/handlers.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index ae15ab05..6a4b3a23 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -372,8 +372,6 @@ func (s *apiV1) handleAddIpfs(c echo.Context, u *util.User) error { func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { ctx := c.Request().Context() - //contentLength := c.Request().ContentLength - if err := util.ErrorIfContentAddingDisabled(s.isContentAddingDisabled(u)); err != nil { return err } @@ -906,9 +904,9 @@ func (s *apiV1) handleGetContent(c echo.Context, u *util.User) error { return err } - //if err := util.IsContentOwner(u.ID, content.UserID); err != nil { - // return err - //} + if err := util.IsContentOwner(u.ID, content.UserID); err != nil { + return err + } return c.JSON(http.StatusOK, content) } From abef0075e9a601292a8170c0b59b41fd740968bc Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 17 May 2023 09:58:07 -0300 Subject: [PATCH 21/25] fixed error handling on no records founds --- api/v1/handlers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index 6a4b3a23..d6e67b3d 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -516,12 +516,13 @@ func (s *apiV1) getUserStorageCapacity(user *util.User) (*util.UsersStorageCapac Scan(&usage).Error; err != nil { return usc, err } + usc.UserId = user.ID usc.Size = usage.TotalSize usc.LastSyncAt = time.Now() s.db.Save(&usc) } - return usc, err + return usc, nil } // handleAdd godoc From 693f98381622e2c4d393a283f6756d15fbe4b606 Mon Sep 17 00:00:00 2001 From: luc Date: Wed, 17 May 2023 10:13:23 -0300 Subject: [PATCH 22/25] added new db table to migrateSchemas() --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 73175713..975d0795 100644 --- a/main.go +++ b/main.go @@ -405,6 +405,7 @@ func migrateSchemas(db *gorm.DB) error { &model.DealQueueTracker{}, &model.SplitQueue{}, &model.SplitQueueTracker{}, + &util.UsersStorageCapacity{}, ); err != nil { return err } From 71e1f3ace3fa28576f754f7acc619b130b9ee233 Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 23 May 2023 09:50:51 -0300 Subject: [PATCH 23/25] change threshold field to have _bytes at the end for clarification --- util/users_storage_capacity.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go index f694ee86..80bbf49a 100644 --- a/util/users_storage_capacity.go +++ b/util/users_storage_capacity.go @@ -10,8 +10,8 @@ type UsersStorageCapacity struct { UserId uint `json:"user_id"` Size int64 `json:"size" gorm:"default:0"` - SoftLimit int64 `json:"soft_limit" gorm:"default:1319413953331"` // Hardlimit*.8 - HardLimit int64 `json:"hard_limit" gorm:"default:1649267441664"` // 1.5TB + SoftLimit int64 `json:"soft_limit_bytes" gorm:"default:1319413953331"` // Hardlimit*.8 + HardLimit int64 `json:"hard_limit_bytes" gorm:"default:1649267441664"` // 1.5TB LastSyncAt time.Time `json:"last_sync_at"` } From c0a4580e6a44703ac025fa96015d50d2698161c4 Mon Sep 17 00:00:00 2001 From: luc Date: Tue, 23 May 2023 09:58:58 -0300 Subject: [PATCH 24/25] updated cut over date for storage calculation --- api/v1/handlers.go | 6 +++--- api/v1/pinning.go | 2 +- util/users_storage_capacity.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/v1/handlers.go b/api/v1/handlers.go index d6e67b3d..3cf24013 100644 --- a/api/v1/handlers.go +++ b/api/v1/handlers.go @@ -390,7 +390,7 @@ func (s *apiV1) handleAddCar(c echo.Context, u *util.User) error { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, - Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed.", util.BytesToTB(usc.HardLimit)), + Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed. We can be reached on the Filecoin slack under the #ecosystem-dev channel.", util.BytesToTB(usc.HardLimit)), } } @@ -582,7 +582,7 @@ func (s *apiV1) handleAdd(c echo.Context, u *util.User) error { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, - Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed.", util.BytesToTB(usc.HardLimit)), + Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed. We can be reached on the Filecoin slack under the #ecosystem-dev channel.", util.BytesToTB(usc.HardLimit)), } } @@ -4588,7 +4588,7 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, - Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed.", util.BytesToTB(usc.HardLimit)), + Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed. We can be reached on the Filecoin slack under the #ecosystem-dev channel.", util.BytesToTB(usc.HardLimit)), } } diff --git a/api/v1/pinning.go b/api/v1/pinning.go index 1107635c..052ae424 100644 --- a/api/v1/pinning.go +++ b/api/v1/pinning.go @@ -272,7 +272,7 @@ func (s *apiV1) handleAddPin(c echo.Context, u *util.User) error { return &util.HttpError{ Code: http.StatusBadRequest, Reason: util.ERR_CONTENT_SIZE_OVER_LIMIT, - Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed.", util.BytesToTB(usc.HardLimit)), + Details: fmt.Sprintf("Reached Max Storage Threshold: %.2f TB. Please contact the Estuary Team for provisionning dedicated infrastruture if additonal storage is needed. We can be reached on the Filecoin slack under the #ecosystem-dev channel.", util.BytesToTB(usc.HardLimit)), } } diff --git a/util/users_storage_capacity.go b/util/users_storage_capacity.go index 80bbf49a..e4160c13 100644 --- a/util/users_storage_capacity.go +++ b/util/users_storage_capacity.go @@ -20,7 +20,7 @@ type Utilization struct { } // CutOverUtilizationDate All content uploaded pass this date will count toward your user storage capacity -const CutOverUtilizationDate = "2023-05-12 00:00:00" +const CutOverUtilizationDate = "2023-05-26 00:00:00" const SyncRefreshInHours = 24 func (usc *UsersStorageCapacity) ValidateThreshold() bool { From 3749074ab925ddf4ddd6784471fb87492aa48cbd Mon Sep 17 00:00:00 2001 From: LucRoy Date: Tue, 23 May 2023 13:00:13 +0000 Subject: [PATCH 25/25] docs: update swagger docs --- docs/docs.go | 32 ++++++++++++++++++++++++++++++++ docs/swagger.json | 32 ++++++++++++++++++++++++++++++++ docs/swagger.yaml | 21 +++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/docs/docs.go b/docs/docs.go index a3273f6d..7e1d79e7 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -3291,6 +3291,38 @@ const docTemplate = `{ } } }, + "/user/utilization": { + "get": { + "description": "This endpoint is used to get utilization stats for the current user.", + "produces": [ + "application/json" + ], + "tags": [ + "User" + ], + "summary": "Gets User Utilization Stats", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/util.HttpError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/util.HttpError" + } + } + } + } + }, "/viewer": { "get": { "description": "This endpoint fetches viewer details such as username, permissions, address, owned miners, user settings etc.", diff --git a/docs/swagger.json b/docs/swagger.json index 697ae5c3..18a2a725 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -3284,6 +3284,38 @@ } } }, + "/user/utilization": { + "get": { + "description": "This endpoint is used to get utilization stats for the current user.", + "produces": [ + "application/json" + ], + "tags": [ + "User" + ], + "summary": "Gets User Utilization Stats", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/util.HttpError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/util.HttpError" + } + } + } + } + }, "/viewer": { "get": { "description": "This endpoint fetches viewer details such as username, permissions, address, owned miners, user settings etc.", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 2f73f7e6..f995d4c6 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2549,6 +2549,27 @@ paths: summary: Get stats for the current user tags: - User + /user/utilization: + get: + description: This endpoint is used to get utilization stats for the current user. + produces: + - application/json + responses: + "200": + description: OK + schema: + type: string + "400": + description: Bad Request + schema: + $ref: '#/definitions/util.HttpError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/util.HttpError' + summary: Gets User Utilization Stats + tags: + - User /viewer: get: description: This endpoint fetches viewer details such as username, permissions, address, owned miners, user settings etc.