Merge pull request #832 from tidepool-org/BACK-3514-out-of-order-bolus-food

[BACK-3514] Handle out-of-order food and bolus data for twiist

Author: darinkrauss

data/data.go

@@ -16,7 +16,8 @@ const (
 )
 
 type SelectorOrigin struct {
-	ID *string `json:"id,omitempty"`
+	ID   *string `json:"id,omitempty" bson:"id,omitempty"`
+	Time *string `json:"time,omitempty" bson:"time,omitempty"` // Inclusive, currently NOT used in database query
 }
 
 func ParseSelectorOrigin(parser structure.ObjectParser) *SelectorOrigin {
@@ -34,15 +35,38 @@ func NewSelectorOrigin() *SelectorOrigin {
 
 func (s *SelectorOrigin) Parse(parser structure.ObjectParser) {
 	s.ID = parser.String("id")
+	s.Time = parser.String("time")
 }
 
 func (s *SelectorOrigin) Validate(validator structure.Validator) {
 	validator.String("id", s.ID).Exists().NotEmpty().LengthLessThanOrEqualTo(SelectorOriginIDLengthMaximum)
+	validator.String("time", s.Time).AsTime(time.RFC3339Nano).NotZero()
+}
+
+func (s *SelectorOrigin) Includes(other *SelectorOrigin) bool {
+	if s == nil || other == nil { // Must not be missing
+		return false
+	} else if s.ID != nil && (other.ID == nil || *s.ID != *other.ID) { // If id matters, then must include
+		return false
+	} else if s.Time == nil { // If time does not matter, success
+		return true
+	} else if other.Time == nil { // Must exist
+		return false
+	} else if sTime, err := time.Parse(time.RFC3339Nano, *s.Time); err != nil || sTime.IsZero() { // Must parse
+		return false
+	} else if otherTime, err := time.Parse(time.RFC3339Nano, *other.Time); err != nil || otherTime.IsZero() { // Must parse
+		return false
+	} else if otherTime.Before(sTime) { // Must include
+		return false
+	} else {
+		return true
+	}
 }
 
 type Selector struct {
-	ID     *string         `json:"id,omitempty"`
-	Origin *SelectorOrigin `json:"origin,omitempty"`
+	ID     *string         `json:"id,omitempty" bson:"id,omitempty"`
+	Time   *time.Time      `json:"time,omitempty" bson:"time,omitempty"` // Inclusive, currently NOT used in database query
+	Origin *SelectorOrigin `json:"origin,omitempty" bson:"origin,omitempty"`
 }
 
 func ParseSelector(parser structure.ObjectParser) *Selector {
@@ -60,6 +84,7 @@ func NewSelector() *Selector {
 
 func (s *Selector) Parse(parser structure.ObjectParser) {
 	s.ID = parser.String("id")
+	s.Time = parser.Time("time", TimeFormat)
 	s.Origin = ParseSelectorOrigin(parser.WithReferenceObjectParser("origin"))
 }
 
@@ -68,11 +93,27 @@ func (s *Selector) Validate(validator structure.Validator) {
 		validator.ReportError(structureValidator.ErrorValuesNotExistForOne("id", "origin"))
 	} else if s.ID != nil {
 		validator.String("id", s.ID).Using(IDValidator)
+		validator.Time("time", s.Time).NotZero()
 	} else {
+		validator.Time("time", s.Time).NotExists()
 		s.Origin.Validate(validator.WithReference("origin"))
 	}
 }
 
+func (s *Selector) Includes(other *Selector) bool {
+	if s == nil || other == nil { // Must not be missing
+		return false
+	} else if s.ID != nil && (other.ID == nil || *s.ID != *other.ID) { // If id matters, then must include
+		return false
+	} else if s.Time != nil && (other.Time == nil || other.Time.Before(*s.Time)) { // If time matters, then must include
+		return false
+	} else if s.Origin != nil && (other.Origin == nil || !s.Origin.Includes(other.Origin)) { // If origin matters, then must include
+		return false
+	} else {
+		return true
+	}
+}
+
 type Selectors []*Selector
 
 func ParseSelectors(parser structure.ArrayParser) *Selectors {
@@ -107,6 +148,16 @@ func (s *Selectors) Validate(validator structure.Validator) {
 	}
 }
 
+func (s *Selectors) Filter(predicate func(*Selector) bool) *Selectors {
+	filtered := Selectors{}
+	for _, selector := range *s {
+		if predicate(selector) {
+			filtered = append(filtered, selector)
+		}
+	}
+	return &filtered
+}
+
 func NewID() string {
 	return id.Must(id.New(16))
 }

data/data_test.go

@@ -1,19 +1,83 @@
 package data_test
 
 import (
+	"time"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	"github.com/tidepool-org/platform/data"
 	errorsTest "github.com/tidepool-org/platform/errors/test"
+	"github.com/tidepool-org/platform/pointer"
 	structureTest "github.com/tidepool-org/platform/structure/test"
 	structureValidator "github.com/tidepool-org/platform/structure/validator"
 	"github.com/tidepool-org/platform/test"
 )
 
 var _ = Describe("Data", func() {
+	Context("SelectorOrigin", func() {
+		Context("Includes", func() {
+			now := time.Now()
+			tm := pointer.FromString(now.Format(time.RFC3339Nano))
+			id := pointer.FromString(data.NewID())
+
+			DescribeTable("return the expected results when the selector origins",
+				func(origin *data.SelectorOrigin, otherOrigin *data.SelectorOrigin, expectedResult bool) {
+					Expect(origin.Includes(otherOrigin)).To(Equal(expectedResult))
+				},
+				Entry("both are nil", nil, nil, false),
+				Entry("origin is nil", nil, &data.SelectorOrigin{}, false),
+				Entry("other origin is nil", &data.SelectorOrigin{}, nil, false),
+				Entry("id and time are nil", &data.SelectorOrigin{}, &data.SelectorOrigin{}, true),
+				Entry("origin id is nil", &data.SelectorOrigin{}, &data.SelectorOrigin{ID: id}, true),
+				Entry("other origin id is nil", &data.SelectorOrigin{ID: id}, &data.SelectorOrigin{}, false),
+				Entry("id mismatch", &data.SelectorOrigin{ID: id}, &data.SelectorOrigin{ID: pointer.FromString("mismatch")}, false),
+				Entry("id includes", &data.SelectorOrigin{ID: id}, &data.SelectorOrigin{ID: id}, true),
+				Entry("origin time is nil", &data.SelectorOrigin{ID: id}, &data.SelectorOrigin{ID: id, Time: tm}, true),
+				Entry("other origin time is nil", &data.SelectorOrigin{ID: id, Time: tm}, &data.SelectorOrigin{ID: id}, false),
+				Entry("time earlier", &data.SelectorOrigin{ID: id, Time: tm}, &data.SelectorOrigin{ID: id, Time: pointer.FromString(now.Add(-time.Hour).Format(time.RFC3339Nano))}, false),
+				Entry("time same", &data.SelectorOrigin{ID: id, Time: tm}, &data.SelectorOrigin{ID: id, Time: tm}, true),
+				Entry("time same in different time zone", &data.SelectorOrigin{ID: id, Time: tm}, &data.SelectorOrigin{ID: id, Time: pointer.FromString(now.In(time.FixedZone("Etc/GMT-1", int(-time.Hour.Seconds()))).Format(time.RFC3339Nano))}, true),
+				Entry("time later", &data.SelectorOrigin{ID: id, Time: tm}, &data.SelectorOrigin{ID: id, Time: pointer.FromString(now.Add(time.Hour).Format(time.RFC3339Nano))}, true),
+			)
+		})
+	})
+
+	Context("Selector", func() {
+		Context("Includes", func() {
+			now := time.Now()
+			tm := pointer.FromTime(now)
+			id := pointer.FromString(data.NewID())
+			originID := pointer.FromString(data.NewID())
+
+			DescribeTable("return the expected results when the selector origins",
+				func(origin *data.Selector, otherOrigin *data.Selector, expectedResult bool) {
+					Expect(origin.Includes(otherOrigin)).To(Equal(expectedResult))
+				},
+				Entry("both are nil", nil, nil, false),
+				Entry("selector is nil", nil, &data.Selector{}, false),
+				Entry("other selector is nil", &data.Selector{}, nil, false),
+				Entry("id, time, and origin are nil", &data.Selector{}, &data.Selector{}, true),
+				Entry("selector id is nil", &data.Selector{}, &data.Selector{ID: id}, true),
+				Entry("other selector id is nil", &data.Selector{ID: id}, &data.Selector{}, false),
+				Entry("id mismatch", &data.Selector{ID: id}, &data.Selector{ID: pointer.FromString("mismatch")}, false),
+				Entry("id includes", &data.Selector{ID: id}, &data.Selector{ID: id}, true),
+				Entry("selector time is nil", &data.Selector{ID: id}, &data.Selector{ID: id, Time: tm}, true),
+				Entry("other selector time is nil", &data.Selector{ID: id, Time: tm}, &data.Selector{ID: id}, false),
+				Entry("time earlier", &data.Selector{ID: id, Time: tm}, &data.Selector{ID: id, Time: pointer.FromTime(now.Add(-time.Hour))}, false),
+				Entry("time same", &data.Selector{ID: id, Time: tm}, &data.Selector{ID: id, Time: tm}, true),
+				Entry("time same in different time zone", &data.Selector{ID: id, Time: tm}, &data.Selector{ID: id, Time: pointer.FromTime(now.In(time.FixedZone("Etc/GMT-1", int(-time.Hour.Seconds()))))}, true),
+				Entry("time later", &data.Selector{ID: id, Time: tm}, &data.Selector{ID: id, Time: pointer.FromTime(now.Add(time.Hour))}, true),
+				Entry("selector origin is nil", &data.Selector{ID: id}, &data.Selector{ID: id, Origin: &data.SelectorOrigin{ID: originID}}, true),
+				Entry("other selector origin is nil", &data.Selector{ID: id, Origin: &data.SelectorOrigin{ID: originID}}, &data.Selector{ID: id}, false),
+				Entry("origin id mismatch", &data.Selector{ID: id, Origin: &data.SelectorOrigin{ID: originID}}, &data.Selector{ID: id, Origin: &data.SelectorOrigin{ID: pointer.FromString("mismatch")}}, false),
+				Entry("origin match", &data.Selector{ID: id, Origin: &data.SelectorOrigin{ID: originID}}, &data.Selector{ID: id, Origin: &data.SelectorOrigin{ID: originID}}, true),
+			)
+		})
+	})
+
 	Context("NewID", func() {
-		It("returns a string of 32 lowercase hexidecimal characters", func() {
+		It("returns a string of 32 lowercase hexadecimal characters", func() {
 			Expect(data.NewID()).To(MatchRegexp("^[0-9a-f]{32}$"))
 		})
 

data/datum.go

@@ -60,6 +60,16 @@ func (d Data) SetModifiedTime(modifiedTime *time.Time) {
 	}
 }
 
+func (d Data) Filter(predicate func(Datum) bool) Data {
+	filtered := Data{}
+	for _, datum := range d {
+		if predicate(datum) {
+			filtered = append(filtered, datum)
+		}
+	}
+	return filtered
+}
+
 // Provenance of a document.
 //
 // Useful for determining additional actions to take. For example, if the

data/deduplicator/deduplicator/data_set_delete_origin.go

@@ -6,24 +6,26 @@ import (
 	"github.com/tidepool-org/platform/data"
 	dataStore "github.com/tidepool-org/platform/data/store"
 	dataTypesUpload "github.com/tidepool-org/platform/data/types/upload"
-	"github.com/tidepool-org/platform/errors"
 	"github.com/tidepool-org/platform/pointer"
 )
 
-const DataSetDeleteOriginName = "org.tidepool.deduplicator.dataset.delete.origin"
+const (
+	DataSetDeleteOriginName    = "org.tidepool.deduplicator.dataset.delete.origin"
+	DataSetDeleteOriginVersion = "1.0.0"
+)
 
 type DataSetDeleteOrigin struct {
-	*Base
+	*DataSetDeleteOriginBase
 }
 
 func NewDataSetDeleteOrigin() (*DataSetDeleteOrigin, error) {
-	base, err := NewBase(DataSetDeleteOriginName, "1.0.0")
+	dataSetDeleteOriginBase, err := NewDataSetDeleteOriginBase(DataSetDeleteOriginName, DataSetDeleteOriginVersion, &dataSetDeleteOriginProvider{})
 	if err != nil {
 		return nil, err
 	}
 
 	return &DataSetDeleteOrigin{
-		Base: base,
+		DataSetDeleteOriginBase: dataSetDeleteOriginBase,
 	}, nil
 }
 
@@ -32,106 +34,23 @@ func (d *DataSetDeleteOrigin) New(ctx context.Context, dataSet *dataTypesUpload.
 }
 
 func (d *DataSetDeleteOrigin) Get(ctx context.Context, dataSet *dataTypesUpload.Upload) (bool, error) {
-	if found, err := d.Base.Get(ctx, dataSet); err != nil || found {
+	if found, err := d.DataSetDeleteOriginBase.Get(ctx, dataSet); err != nil || found {
 		return found, err
 	}
 
 	return dataSet.HasDeduplicatorNameMatch("org.tidepool.continuous.origin"), nil // TODO: DEPRECATED
 }
 
-func (d *DataSetDeleteOrigin) Open(ctx context.Context, repository dataStore.DataRepository, dataSet *dataTypesUpload.Upload) (*dataTypesUpload.Upload, error) {
-	if ctx == nil {
-		return nil, errors.New("context is missing")
-	}
-	if repository == nil {
-		return nil, errors.New("repository is missing")
-	}
-	if dataSet == nil {
-		return nil, errors.New("data set is missing")
-	}
-
-	if dataSet.HasDataSetTypeContinuous() {
-		dataSet.SetActive(true)
-	}
-
-	return d.Base.Open(ctx, repository, dataSet)
-}
-
-func (d *DataSetDeleteOrigin) AddData(ctx context.Context, repository dataStore.DataRepository, dataSet *dataTypesUpload.Upload, dataSetData data.Data) error {
-	if ctx == nil {
-		return errors.New("context is missing")
-	}
-	if repository == nil {
-		return errors.New("repository is missing")
-	}
-	if dataSet == nil {
-		return errors.New("data set is missing")
-	}
-	if dataSetData == nil {
-		return errors.New("data set data is missing")
-	}
-
-	if dataSet.HasDataSetTypeContinuous() {
-		dataSetData.SetActive(true)
-	}
-
-	if selectors := d.getSelectors(dataSetData); selectors != nil {
-		if err := repository.DeleteDataSetData(ctx, dataSet, selectors); err != nil {
-			return err
-		}
-		if err := d.Base.AddData(ctx, repository, dataSet, dataSetData); err != nil {
-			return err
-		}
-		return repository.DestroyDeletedDataSetData(ctx, dataSet, selectors)
-	}
+type dataSetDeleteOriginProvider struct{}
 
-	return d.Base.AddData(ctx, repository, dataSet, dataSetData)
+func (d *dataSetDeleteOriginProvider) FilterData(ctx context.Context, repository dataStore.DataRepository, dataSet *dataTypesUpload.Upload, dataSetData data.Data) (data.Data, error) {
+	return dataSetData, nil
 }
 
-func (d *DataSetDeleteOrigin) DeleteData(ctx context.Context, repository dataStore.DataRepository, dataSet *dataTypesUpload.Upload, selectors *data.Selectors) error {
-	if ctx == nil {
-		return errors.New("context is missing")
-	}
-	if repository == nil {
-		return errors.New("repository is missing")
-	}
-	if dataSet == nil {
-		return errors.New("data set is missing")
-	}
-	if selectors == nil {
-		return errors.New("selectors is missing")
-	}
-
-	return repository.ArchiveDataSetData(ctx, dataSet, selectors)
-}
-
-func (d *DataSetDeleteOrigin) Close(ctx context.Context, repository dataStore.DataRepository, dataSet *dataTypesUpload.Upload) error {
-	if ctx == nil {
-		return errors.New("context is missing")
-	}
-	if repository == nil {
-		return errors.New("repository is missing")
-	}
-	if dataSet == nil {
-		return errors.New("data set is missing")
-	}
-
-	if dataSet.HasDataSetTypeContinuous() {
-		return nil
-	}
-
-	return d.Base.Close(ctx, repository, dataSet)
-}
-
-func (d *DataSetDeleteOrigin) getSelectors(dataSetData data.Data) *data.Selectors {
+func (d *dataSetDeleteOriginProvider) GetDataSelectors(dataSetData data.Data) *data.Selectors {
 	selectors := data.Selectors{}
 	for _, dataSetDatum := range dataSetData {
-		if origin := dataSetDatum.GetOrigin(); origin != nil && origin.ID != nil {
-			selector := &data.Selector{
-				Origin: &data.SelectorOrigin{
-					ID: pointer.CloneString(origin.ID),
-				},
-			}
+		if selector := d.getDatumSelector(dataSetDatum); selector != nil {
 			selectors = append(selectors, selector)
 		}
 	}
@@ -140,3 +59,14 @@ func (d *DataSetDeleteOrigin) getSelectors(dataSetData data.Data) *data.Selector
 	}
 	return &selectors
 }
+
+func (d *dataSetDeleteOriginProvider) getDatumSelector(dataSetDatum data.Datum) *data.Selector {
+	if origin := dataSetDatum.GetOrigin(); origin != nil && origin.ID != nil {
+		return &data.Selector{
+			Origin: &data.SelectorOrigin{
+				ID: pointer.CloneString(origin.ID),
+			},
+		}
+	}
+	return nil
+}