From ef07ca80fd26fd35ad013d3abeec422044727e4c Mon Sep 17 00:00:00 2001 From: Pang Wu Date: Wed, 15 Apr 2015 23:16:31 -0700 Subject: [PATCH 1/5] Fix first cluster search --- .../suripu/algorithm/sleep/SleepPeriod.java | 20 ++- .../hello/suripu/algorithm/sleep/Vote.java | 120 ++++++++++++++---- .../suripu/algorithm/sleep/VotingSegment.java | 14 ++ .../suripu/algorithm/utils/DataUtils.java | 25 ++++ 4 files changed, 152 insertions(+), 27 deletions(-) create mode 100644 suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/VotingSegment.java diff --git a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/SleepPeriod.java b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/SleepPeriod.java index 0d4719cba..5898ffbd2 100644 --- a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/SleepPeriod.java +++ b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/SleepPeriod.java @@ -83,10 +83,22 @@ private boolean isAwake(final long startMillis, final long endMillis){ return false; } - public List getAwakePeriods(final boolean debug){ + private double getVote(final long startMillis, final long endMillis){ + + double maxVote = 0; + for(int i = 0; i < this.votes.size(); i++){ + final Pair vote = this.votes.get(i); + if(vote.getFirst() >= startMillis && vote.getFirst() <= endMillis && vote.getSecond() > maxVote){ + maxVote = vote.getSecond(); + } + } + return maxVote; + } + + public List getAwakePeriods(final boolean debug){ long startMillis = 0; long endMillis = 0; - final List result = new ArrayList<>(); + final List result = new ArrayList<>(); for(int i = 0; i < this.votes.size(); i++){ final Pair vote = this.votes.get(i); @@ -97,7 +109,7 @@ public List getAwakePeriods(final boolean debug){ endMillis = vote.getFirst(); }else { if(isAwake(startMillis, endMillis)){ - result.add(new Segment(startMillis, endMillis, this.getOffsetMillis())); + result.add(new VotingSegment(startMillis, endMillis, this.getOffsetMillis(), getVote(startMillis, endMillis))); if(debug){ LOGGER.debug("User awake at {} - {}", new DateTime(startMillis, DateTimeZone.forOffsetMillis(this.getOffsetMillis())), @@ -110,7 +122,7 @@ public List getAwakePeriods(final boolean debug){ } if(isAwake(startMillis, endMillis)){ - result.add(new Segment(startMillis, endMillis, this.getOffsetMillis())); + result.add(new VotingSegment(startMillis, endMillis, this.getOffsetMillis(), getVote(startMillis, endMillis))); if(debug){ LOGGER.debug("User awake at {} - {}", new DateTime(startMillis, DateTimeZone.forOffsetMillis(this.getOffsetMillis())), diff --git a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java index 4189e5da2..26c481c3a 100644 --- a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java +++ b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java @@ -30,6 +30,7 @@ public class Vote { private final MotionScoreAlgorithm motionScoreAlgorithmDefault; private final MotionCluster motionCluster; + private final List alignedAmplitude; private final Map> aggregatedFeatures; private final double rawAmpMean; private final double rawKickOffMean; @@ -62,6 +63,7 @@ public Vote(final List rawData, this.rawKickOffMean = NumericalUtils.mean(noDuplicateKickOffCounts); List dataWithGapFilled = DataUtils.fillMissingValues(noDuplicates, DateTimeConstants.MILLIS_PER_MINUTE); + this.alignedAmplitude = Lists.newArrayList(dataWithGapFilled); List alignedKickOffs = DataUtils.fillMissingValues(noDuplicateKickOffCounts, DateTimeConstants.MILLIS_PER_MINUTE); if(insertEmpty) { final int insertLengthMin = 20; @@ -116,22 +118,26 @@ public Vote(final List rawData, public final List getAwakes(final long fallAsleepMillis, final long wakeUpMillis, final boolean debug){ - final List allAwakesPeriods = this.sleepPeriod.getAwakePeriods(debug); - final List awakesInTheRange = new ArrayList<>(); + final List allAwakesPeriods = this.sleepPeriod.getAwakePeriods(debug); + final List awakesInTheRange = new ArrayList<>(); - for(final Segment segment:allAwakesPeriods){ + for(final VotingSegment segment:allAwakesPeriods){ if(fallAsleepMillis <= segment.getStartTimestamp() && wakeUpMillis >= segment.getEndTimestamp()){ awakesInTheRange.add(segment); } } - final List smoothedAwakes = smoothAwakes(awakesInTheRange, MotionCluster.toSegments(this.motionCluster.getCopyOfClusters())); - return smoothedAwakes; + final List smoothedAwakes = smoothAwakes(awakesInTheRange, MotionCluster.toSegments(this.motionCluster.getCopyOfClusters())); + final List results = new ArrayList<>(); + for(final VotingSegment votingSegment:smoothedAwakes){ + results.add(votingSegment); + } + return results; } - private List getAwakesInTimeSpanMillis(final List awakes, final long startMillis, final long endMillis){ - final List result = new ArrayList<>(); - for(final Segment awake:awakes){ + private List getAwakesInTimeSpanMillis(final List awakes, final long startMillis, final long endMillis){ + final List result = new ArrayList<>(); + for(final VotingSegment awake:awakes){ if(awake.getStartTimestamp() >= startMillis && awake.getEndTimestamp() <= endMillis){ result.add(awake); } @@ -140,13 +146,13 @@ private List getAwakesInTimeSpanMillis(final List awakes, fina return result; } - private List filterAwakeFragments(final List awakesInMotionCluster){ - final List result = new ArrayList<>(); + private List filterAwakeFragments(final List awakesInMotionCluster){ + final List result = new ArrayList<>(); if(awakesInMotionCluster.size() < 2){ return awakesInMotionCluster; } - for(final Segment awake:awakesInMotionCluster){ + for(final VotingSegment awake:awakesInMotionCluster){ if(awake.getDuration() < 20 * DateTimeConstants.MILLIS_PER_MINUTE){ continue; } @@ -160,10 +166,10 @@ private List filterAwakeFragments(final List awakesInMotionClu * Idea: if multiple awakes is in the same motion cluster, filter out those less than * 20 minutes, less is more. */ - private List smoothAwakes(final List awakes, final List motionClusters){ - final List smoothedAwakes = new ArrayList<>(); + private List smoothAwakes(final List awakes, final List motionClusters){ + final List smoothedAwakes = new ArrayList<>(); for(final Segment currentCluster:motionClusters){ - final List awakesInMotionCluster = getAwakesInTimeSpanMillis(awakes, + final List awakesInMotionCluster = getAwakesInTimeSpanMillis(awakes, currentCluster.getStartTimestamp(), currentCluster.getEndTimestamp()); smoothedAwakes.addAll(filterAwakeFragments(awakesInMotionCluster)); @@ -258,11 +264,10 @@ private SleepEvents aggregate(final SleepEvents defaultEvents) clusterCopy, this.getAggregatedFeatures(), defaultEvents.fallAsleep.getStartTimestamp()); - long sleepTimeMillis = sleepTimesMillis.getSecond(); - inBed = new Segment(sleepTimesMillis.getFirst(), sleepTimesMillis.getFirst() + DateTimeConstants.MILLIS_PER_MINUTE, sleep.getOffsetMillis()); - sleep = new Segment(sleepTimeMillis, - sleepTimeMillis + DateTimeConstants.MILLIS_PER_MINUTE, - defaultEvents.fallAsleep.getOffsetMillis()); + long sleepTimeMillis = DataUtils.findNearestDataTime(this.alignedAmplitude, sleepTimesMillis.getSecond()); + long inBedTimeMillis = DataUtils.findNearestDataTime(this.alignedAmplitude, sleepTimesMillis.getFirst()); + inBed = new Segment(inBedTimeMillis, inBedTimeMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.goToBed.getOffsetMillis()); + sleep = new Segment(sleepTimeMillis, sleepTimeMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.fallAsleep.getOffsetMillis()); if(inBed.getStartTimestamp() > sleep.getStartTimestamp()){ inBed = new Segment(sleep.getStartTimestamp() - 10 * DateTimeConstants.MILLIS_PER_MINUTE, sleep.getStartTimestamp() - 9 * DateTimeConstants.MILLIS_PER_MINUTE, @@ -273,12 +278,19 @@ private SleepEvents aggregate(final SleepEvents defaultEvents) Segment wakeUp = defaultEvents.wakeUp; Segment outBed = defaultEvents.outOfBed; - final Pair wakeUpTimesMillis = safeGuardPickWakeUp(clusterCopy, + Pair wakeUpTimesMillis = safeGuardPickWakeUp(clusterCopy, sleepPeriod, getAggregatedFeatures(), defaultEvents.wakeUp.getStartTimestamp()); - wakeUp = new Segment(wakeUpTimesMillis.getFirst(), wakeUpTimesMillis.getFirst() + DateTimeConstants.MILLIS_PER_MINUTE, wakeUp.getOffsetMillis()); - outBed = new Segment(wakeUpTimesMillis.getSecond(), wakeUpTimesMillis.getSecond() + DateTimeConstants.MILLIS_PER_MINUTE, inBed.getOffsetMillis()); + /*wakeUpTimesMillis = votingSafeGuardPickWakeUp(this.sleepPeriod.getAwakePeriods(false), + sleepPeriod, + getAggregatedFeatures(), + wakeUpTimesMillis);*/ + + long wakeUpMillis = DataUtils.findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getFirst()); + long outBedMillis = DataUtils.findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getSecond()); + wakeUp = new Segment(wakeUpMillis, wakeUpMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.wakeUp.getOffsetMillis()); + outBed = new Segment(outBedMillis, outBedMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.outOfBed.getOffsetMillis()); return SleepEvents.create(inBed, sleep, wakeUp, outBed); @@ -292,6 +304,54 @@ private static Pair predictionBoundsMillis(final long wakeUpMillisPr return new Pair<>(wakeUpMillisPredicted, predictionSegment.get().getEndTimestamp()); } + private static Optional getLastAwakeInSleepPeriod(final List awakesUnfiltered, + final SleepPeriod sleepPeriod){ + Optional lastAwake = Optional.absent(); + for(final VotingSegment votingSegment:awakesUnfiltered){ + if(votingSegment.getStartTimestamp() <= sleepPeriod.getEndTimestamp()){ + lastAwake = Optional.of(votingSegment); + } + } + + return lastAwake; + } + + protected static Pair votingSafeGuardPickWakeUp(final List awakesUnfiltered, + final SleepPeriod sleepPeriod, + final Map> featuresNotCapped, + final Pair wakeUpTimesSafeGuarded){ + final Optional lastAwakeInSleepPeriod = getLastAwakeInSleepPeriod(awakesUnfiltered, sleepPeriod); + if(!lastAwakeInSleepPeriod.isPresent()){ + return wakeUpTimesSafeGuarded; + } + + if(lastAwakeInSleepPeriod.get().getStartTimestamp() - wakeUpTimesSafeGuarded.getFirst() < 15 * DateTimeConstants.MILLIS_PER_MINUTE){ + LOGGER.debug("HAPPY USER3: Last wake up close to safeguarded result, last wake up {} - {}", + new DateTime(lastAwakeInSleepPeriod.get().getStartTimestamp(), + DateTimeZone.forOffsetMillis(lastAwakeInSleepPeriod.get().getOffsetMillis())), + new DateTime(lastAwakeInSleepPeriod.get().getEndTimestamp(), + DateTimeZone.forOffsetMillis(lastAwakeInSleepPeriod.get().getOffsetMillis()))); + return wakeUpTimesSafeGuarded; + } + final Optional maxScore = getMaxScore(featuresNotCapped, + MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, + lastAwakeInSleepPeriod.get().getStartTimestamp(), + lastAwakeInSleepPeriod.get().getEndTimestamp()); + + LOGGER.debug("Wrong safeguarding: Last wake up far away from safeguarded result, last wake up {} - {}", + new DateTime(lastAwakeInSleepPeriod.get().getStartTimestamp(), + DateTimeZone.forOffsetMillis(lastAwakeInSleepPeriod.get().getOffsetMillis())), + new DateTime(lastAwakeInSleepPeriod.get().getEndTimestamp(), + DateTimeZone.forOffsetMillis(lastAwakeInSleepPeriod.get().getOffsetMillis()))); + + if(!maxScore.isPresent()){ + return new Pair(lastAwakeInSleepPeriod.get().getStartTimestamp(), + Math.min(lastAwakeInSleepPeriod.get().getEndTimestamp(), sleepPeriod.getEndTimestamp())); + } + return new Pair(maxScore.get().timestamp, + Math.max(lastAwakeInSleepPeriod.get().getEndTimestamp(), sleepPeriod.getEndTimestamp())); + } + protected static Pair safeGuardPickWakeUp(final List clusters, final SleepPeriod sleepPeriod, final Map> featuresNotCapped, @@ -327,6 +387,20 @@ protected static Pair safeGuardPickWakeUp(final List maxMotionScore = getMaxScore(featuresNotCapped, + MotionFeatures.FeatureType.MAX_AMPLITUDE, + wakeUpMillisPredicted, + sleepPeriod.getEndTimestamp()); + + if(maxMotionScore.isPresent() && maxSleepScoreOptional.isPresent() && maxMotionScore.get().timestamp > maxSleepScoreOptional.get().timestamp){ + final Optional maxMotionClusterOptional = getClusterByTimeMillis(clusterSegments, maxMotionScore.get().timestamp, 0, 0); + if(!maxMotionClusterOptional.isPresent()){ + return new Pair(maxMotionScore.get().timestamp, maxMotionScore.get().timestamp + 10 * DateTimeConstants.MILLIS_PER_MINUTE); + } + return new Pair(maxMotionScore.get().timestamp, maxMotionClusterOptional.get().getEndTimestamp()); + } + */ if(maxSleepScoreOptional.isPresent()){ if(wakeUpMillisPredicted <= maxSleepScoreOptional.get().timestamp && @@ -508,7 +582,7 @@ protected static Pair safeGuardPickSleep(final SleepPeriod sleepPeri final Optional firstMaxScoreItemOptional = getMaxScore(features, MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, - firstCluster.getStartTimestamp(), + sleepPeriod.getStartTimestamp(), firstCluster.getEndTimestamp() + 20 * DateTimeConstants.MILLIS_PER_MINUTE); final Optional maxScoreSinceSleep = getMaxScore(features, MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, diff --git a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/VotingSegment.java b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/VotingSegment.java new file mode 100644 index 000000000..c4ea580d8 --- /dev/null +++ b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/VotingSegment.java @@ -0,0 +1,14 @@ +package com.hello.suripu.algorithm.sleep; + +import com.hello.suripu.algorithm.core.Segment; + +/** + * Created by pangwu on 4/14/15. + */ +public class VotingSegment extends Segment { + public final double vote; + public VotingSegment(final long startTimestampMillis, final long endTimestampMillis, final int offsetMillis, final double vote){ + super(startTimestampMillis, endTimestampMillis, offsetMillis); + this.vote = vote; + } +} diff --git a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/utils/DataUtils.java b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/utils/DataUtils.java index 03a0027a0..ac40a6a6d 100644 --- a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/utils/DataUtils.java +++ b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/utils/DataUtils.java @@ -116,4 +116,29 @@ public static List insertEmptyData(final List data result.addAll(data); return result; } + + public static long findNearestDataTime(final List data, final long targetMillis){ + int minDiff = Integer.MAX_VALUE; + long time = 0; + for(int i = 0; i < data.size(); i++){ + if(data.get(i).amplitude == 0){ + continue; + } + + final int diff = (int) Math.abs(data.get(i).timestamp - targetMillis); + if(diff < minDiff){ + minDiff = diff; + time = data.get(i).timestamp; + } + } + + if(minDiff == Integer.MAX_VALUE){ + return targetMillis; + } + + /*if(Math.abs(time - targetMillis) > 15 * DateTimeConstants.MILLIS_PER_MINUTE){ + return targetMillis; + }*/ + return time; + } } From cf2910d8e3516a708382e54dfb087ec48f946ddd Mon Sep 17 00:00:00 2001 From: Pang Wu Date: Wed, 15 Apr 2015 23:24:36 -0700 Subject: [PATCH 2/5] WIP --- .../main/java/com/hello/suripu/algorithm/sleep/Vote.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java index 26c481c3a..6d72c3a82 100644 --- a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java +++ b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java @@ -582,7 +582,7 @@ protected static Pair safeGuardPickSleep(final SleepPeriod sleepPeri final Optional firstMaxScoreItemOptional = getMaxScore(features, MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, - sleepPeriod.getStartTimestamp(), + firstCluster.getStartTimestamp(), firstCluster.getEndTimestamp() + 20 * DateTimeConstants.MILLIS_PER_MINUTE); final Optional maxScoreSinceSleep = getMaxScore(features, MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, @@ -614,7 +614,7 @@ protected static Pair safeGuardPickSleep(final SleepPeriod sleepPeri if(predictedMaxScoreOptional.isPresent() && firstMaxScoreItemOptional.isPresent()){ - if(predictedMaxScoreOptional.get().amplitude / 5d > firstMaxScoreItemOptional.get().amplitude){ + if(originalSleepMillis >= sleepPeriod.getStartTimestamp() && predictedMaxScoreOptional.get().amplitude / 5d > firstMaxScoreItemOptional.get().amplitude){ final Optional maxScoreCluster = getClusterByTimeMillis(clusterSegments, predictedMaxScoreOptional.get().timestamp, 0, 0); @@ -628,6 +628,11 @@ protected static Pair safeGuardPickSleep(final SleepPeriod sleepPeri } } + + if(originalSleepMillis < sleepPeriod.getStartTimestamp()){ + LOGGER.debug("NOISY OUT OF BOUND: fallback to 1st cluster in sleep period"); + return new Pair(firstCluster.getStartTimestamp(), firstMaxScoreItemOptional.get().timestamp); + } } LOGGER.debug("COMPLETELY OFF: Move prediction to first cluster {} - {}", From 15c11d6c7db4d555ae51c53f2fa04dfe720007e0 Mon Sep 17 00:00:00 2001 From: Pang Wu Date: Fri, 17 Apr 2015 00:51:01 -0700 Subject: [PATCH 3/5] check point --- .../hello/suripu/algorithm/sleep/Vote.java | 111 ++++++++++-------- .../suripu/algorithm/utils/DataUtils.java | 25 ---- 2 files changed, 65 insertions(+), 71 deletions(-) diff --git a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java index 6d72c3a82..5aae96941 100644 --- a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java +++ b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java @@ -264,8 +264,8 @@ private SleepEvents aggregate(final SleepEvents defaultEvents) clusterCopy, this.getAggregatedFeatures(), defaultEvents.fallAsleep.getStartTimestamp()); - long sleepTimeMillis = DataUtils.findNearestDataTime(this.alignedAmplitude, sleepTimesMillis.getSecond()); - long inBedTimeMillis = DataUtils.findNearestDataTime(this.alignedAmplitude, sleepTimesMillis.getFirst()); + long sleepTimeMillis = findNearestDataTime(this.alignedAmplitude, sleepTimesMillis.getSecond()); + long inBedTimeMillis = findNearestDataTime(this.alignedAmplitude, sleepTimesMillis.getFirst()); inBed = new Segment(inBedTimeMillis, inBedTimeMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.goToBed.getOffsetMillis()); sleep = new Segment(sleepTimeMillis, sleepTimeMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.fallAsleep.getOffsetMillis()); if(inBed.getStartTimestamp() > sleep.getStartTimestamp()){ @@ -282,13 +282,14 @@ private SleepEvents aggregate(final SleepEvents defaultEvents) sleepPeriod, getAggregatedFeatures(), defaultEvents.wakeUp.getStartTimestamp()); - /*wakeUpTimesMillis = votingSafeGuardPickWakeUp(this.sleepPeriod.getAwakePeriods(false), + wakeUpTimesMillis = votingSafeGuardPickWakeUp(this.sleepPeriod.getAwakePeriods(false), sleepPeriod, getAggregatedFeatures(), - wakeUpTimesMillis);*/ + wakeUpTimesMillis, + defaultEvents.wakeUp.getStartTimestamp()); - long wakeUpMillis = DataUtils.findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getFirst()); - long outBedMillis = DataUtils.findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getSecond()); + long wakeUpMillis = findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getFirst()); + long outBedMillis = findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getSecond()); wakeUp = new Segment(wakeUpMillis, wakeUpMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.wakeUp.getOffsetMillis()); outBed = new Segment(outBedMillis, outBedMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.outOfBed.getOffsetMillis()); @@ -296,6 +297,28 @@ private SleepEvents aggregate(final SleepEvents defaultEvents) return SleepEvents.create(inBed, sleep, wakeUp, outBed); } + private long findNearestDataTime(final List data, final long targetMillis){ + int minDiff = Integer.MAX_VALUE; + long time = 0; + for(int i = 0; i < data.size(); i++){ + if(data.get(i).amplitude == 0){ + continue; + } + + final int diff = (int) Math.abs(data.get(i).timestamp - targetMillis); + if(diff < minDiff){ + minDiff = diff; + time = data.get(i).timestamp; + } + } + + if(minDiff == Integer.MAX_VALUE){ + return targetMillis; + } + + return time; + } + private static Pair predictionBoundsMillis(final long wakeUpMillisPredicted, final Optional predictionSegment){ if(!predictionSegment.isPresent()){ @@ -304,23 +327,37 @@ private static Pair predictionBoundsMillis(final long wakeUpMillisPr return new Pair<>(wakeUpMillisPredicted, predictionSegment.get().getEndTimestamp()); } - private static Optional getLastAwakeInSleepPeriod(final List awakesUnfiltered, - final SleepPeriod sleepPeriod){ - Optional lastAwake = Optional.absent(); + private static Optional getNextAwakeInSleepPeriod(final List awakesUnfiltered, + final SleepPeriod sleepPeriod, + final long wakeUpSafeGuarded){ for(final VotingSegment votingSegment:awakesUnfiltered){ - if(votingSegment.getStartTimestamp() <= sleepPeriod.getEndTimestamp()){ - lastAwake = Optional.of(votingSegment); + if(votingSegment.getStartTimestamp() > sleepPeriod.getEndTimestamp()){ + continue; + } + + if(votingSegment.getDuration() < 20 * DateTimeConstants.MILLIS_PER_MINUTE){ + continue; + } + + if(votingSegment.getStartTimestamp() >= wakeUpSafeGuarded || + (votingSegment.getStartTimestamp() < wakeUpSafeGuarded && votingSegment.getEndTimestamp() >= wakeUpSafeGuarded)){ + return Optional.of(votingSegment); } } - return lastAwake; + return Optional.absent(); } protected static Pair votingSafeGuardPickWakeUp(final List awakesUnfiltered, final SleepPeriod sleepPeriod, final Map> featuresNotCapped, - final Pair wakeUpTimesSafeGuarded){ - final Optional lastAwakeInSleepPeriod = getLastAwakeInSleepPeriod(awakesUnfiltered, sleepPeriod); + final Pair wakeUpTimesSafeGuarded, + final long wakeUpMillisPredicted){ + if(wakeUpMillisPredicted < wakeUpTimesSafeGuarded.getFirst()){ + return wakeUpTimesSafeGuarded; + } + + final Optional lastAwakeInSleepPeriod = getNextAwakeInSleepPeriod(awakesUnfiltered, sleepPeriod, wakeUpTimesSafeGuarded.getFirst()); if(!lastAwakeInSleepPeriod.isPresent()){ return wakeUpTimesSafeGuarded; } @@ -345,11 +382,9 @@ protected static Pair votingSafeGuardPickWakeUp(final List(lastAwakeInSleepPeriod.get().getStartTimestamp(), - Math.min(lastAwakeInSleepPeriod.get().getEndTimestamp(), sleepPeriod.getEndTimestamp())); + return new Pair<>(lastAwakeInSleepPeriod.get().getStartTimestamp(), lastAwakeInSleepPeriod.get().getEndTimestamp()); } - return new Pair(maxScore.get().timestamp, - Math.max(lastAwakeInSleepPeriod.get().getEndTimestamp(), sleepPeriod.getEndTimestamp())); + return new Pair<>(maxScore.get().timestamp, lastAwakeInSleepPeriod.get().getEndTimestamp()); } protected static Pair safeGuardPickWakeUp(final List clusters, @@ -373,8 +408,10 @@ protected static Pair safeGuardPickWakeUp(final List(wakeUpMillisPredicted, lastSegmentInSleepPeriod.getEndTimestamp()); } + // predict < last cluster if(wakeUpMillisPredicted < lastSegmentInSleepPeriod.getStartTimestamp()) { - // predict << last segment of sleep period + // predict << last segment of sleep period, this user toss and turn a lot during sleep + // The prediction can landed on one of the heavy toss-and-turn cluster. if (lastSegmentInSleepPeriod.getStartTimestamp() - wakeUpMillisPredicted > 40 * DateTimeConstants.MILLIS_PER_MINUTE) { LOGGER.debug("Predicted too far way from end, predicted {}", new DateTime(wakeUpMillisPredicted, DateTimeZone.forOffsetMillis(lastSegment.getOffsetMillis()))); @@ -387,22 +424,8 @@ protected static Pair safeGuardPickWakeUp(final List maxMotionScore = getMaxScore(featuresNotCapped, - MotionFeatures.FeatureType.MAX_AMPLITUDE, - wakeUpMillisPredicted, - sleepPeriod.getEndTimestamp()); - - if(maxMotionScore.isPresent() && maxSleepScoreOptional.isPresent() && maxMotionScore.get().timestamp > maxSleepScoreOptional.get().timestamp){ - final Optional maxMotionClusterOptional = getClusterByTimeMillis(clusterSegments, maxMotionScore.get().timestamp, 0, 0); - if(!maxMotionClusterOptional.isPresent()){ - return new Pair(maxMotionScore.get().timestamp, maxMotionScore.get().timestamp + 10 * DateTimeConstants.MILLIS_PER_MINUTE); - } - return new Pair(maxMotionScore.get().timestamp, maxMotionClusterOptional.get().getEndTimestamp()); - } - */ - - if(maxSleepScoreOptional.isPresent()){ + if(maxSleepScoreOptional.isPresent()){ // deal with edge case, noo significant motion. + // decide if we should safe guard this result to the max score end if(wakeUpMillisPredicted <= maxSleepScoreOptional.get().timestamp && maxSleepScoreOptional.get().timestamp < lastSegmentInSleepPeriod.getStartTimestamp()){ LOGGER.debug("Max drop between prediction and last segment detected, prediction is likely right."); @@ -420,7 +443,8 @@ protected static Pair safeGuardPickWakeUp(final List(maxWakeUpScoreOptional.get().timestamp, clusters.get(maxScoreCluster.getSecond()).timestamp); }else { - + // Prediction is nearby the last cluster, but still not the last + // fallback to the moment has max wake up score. LOGGER.debug("OK USER: Predict not too far from end"); final Optional maxWakeUpScoreOptional = getMaxScore(featuresNotCapped, MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, @@ -439,11 +463,11 @@ protected static Pair safeGuardPickWakeUp(final List last segment in sleep period + // predict > last segment in sleep period, possible caused by maid or wrong sleep period detection if(lastSegment.getStartTimestamp() == lastSegmentInSleepPeriod.getStartTimestamp() && lastSegment.getEndTimestamp() == lastSegmentInSleepPeriod.getEndTimestamp()){ - // No motion cluster after end of sleep period. + // No motion cluster after end of sleep period. error not caused by maid motion LOGGER.debug("-------------* No maid found, last motion cluster {} - {}", new DateTime(lastSegment.getStartTimestamp(), DateTimeZone.forOffsetMillis(lastSegment.getOffsetMillis())), new DateTime(lastSegment.getEndTimestamp(), DateTimeZone.forOffsetMillis(lastSegment.getOffsetMillis()))); @@ -464,7 +488,7 @@ protected static Pair safeGuardPickWakeUp(final List(wakeUpMillisPredicted, lastMotionMillis); }else { - + // Maid or partner motion found, maid might cause multiple motion clusters // last segment in sleep period < last segment && predict > last segment in sleep period LOGGER.debug("-------------* Maid found, last motion cluster in sleep period {} - {}", new DateTime(lastSegmentInSleepPeriod.getStartTimestamp(), DateTimeZone.forOffsetMillis(lastSegmentInSleepPeriod.getOffsetMillis())), @@ -477,7 +501,7 @@ protected static Pair safeGuardPickWakeUp(final List maxWakeUpScoreOptional = getMaxScore(featuresNotCapped, MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, lastSegmentInSleepPeriod.getEndTimestamp() - 60 * DateTimeConstants.MILLIS_PER_MINUTE, @@ -614,7 +638,7 @@ protected static Pair safeGuardPickSleep(final SleepPeriod sleepPeri if(predictedMaxScoreOptional.isPresent() && firstMaxScoreItemOptional.isPresent()){ - if(originalSleepMillis >= sleepPeriod.getStartTimestamp() && predictedMaxScoreOptional.get().amplitude / 5d > firstMaxScoreItemOptional.get().amplitude){ + if(predictedMaxScoreOptional.get().amplitude / 5d > firstMaxScoreItemOptional.get().amplitude){ final Optional maxScoreCluster = getClusterByTimeMillis(clusterSegments, predictedMaxScoreOptional.get().timestamp, 0, 0); @@ -628,11 +652,6 @@ protected static Pair safeGuardPickSleep(final SleepPeriod sleepPeri } } - - if(originalSleepMillis < sleepPeriod.getStartTimestamp()){ - LOGGER.debug("NOISY OUT OF BOUND: fallback to 1st cluster in sleep period"); - return new Pair(firstCluster.getStartTimestamp(), firstMaxScoreItemOptional.get().timestamp); - } } LOGGER.debug("COMPLETELY OFF: Move prediction to first cluster {} - {}", diff --git a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/utils/DataUtils.java b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/utils/DataUtils.java index ac40a6a6d..03a0027a0 100644 --- a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/utils/DataUtils.java +++ b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/utils/DataUtils.java @@ -116,29 +116,4 @@ public static List insertEmptyData(final List data result.addAll(data); return result; } - - public static long findNearestDataTime(final List data, final long targetMillis){ - int minDiff = Integer.MAX_VALUE; - long time = 0; - for(int i = 0; i < data.size(); i++){ - if(data.get(i).amplitude == 0){ - continue; - } - - final int diff = (int) Math.abs(data.get(i).timestamp - targetMillis); - if(diff < minDiff){ - minDiff = diff; - time = data.get(i).timestamp; - } - } - - if(minDiff == Integer.MAX_VALUE){ - return targetMillis; - } - - /*if(Math.abs(time - targetMillis) > 15 * DateTimeConstants.MILLIS_PER_MINUTE){ - return targetMillis; - }*/ - return time; - } } From 3da3c0b08cd0909f9cca848c9ac16f75b4a49dc1 Mon Sep 17 00:00:00 2001 From: Pang Wu Date: Fri, 17 Apr 2015 15:04:24 -0700 Subject: [PATCH 4/5] WIP --- .../hello/suripu/algorithm/sleep/Vote.java | 74 ++++++++++++++----- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java index 5aae96941..302aa4c64 100644 --- a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java +++ b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java @@ -153,7 +153,7 @@ private List filterAwakeFragments(final List awake } for(final VotingSegment awake:awakesInMotionCluster){ - if(awake.getDuration() < 20 * DateTimeConstants.MILLIS_PER_MINUTE){ + if(awake.getDuration() < 10 * DateTimeConstants.MILLIS_PER_MINUTE){ continue; } result.add(awake); @@ -283,6 +283,7 @@ private SleepEvents aggregate(final SleepEvents defaultEvents) getAggregatedFeatures(), defaultEvents.wakeUp.getStartTimestamp()); wakeUpTimesMillis = votingSafeGuardPickWakeUp(this.sleepPeriod.getAwakePeriods(false), + clusterCopy, sleepPeriod, getAggregatedFeatures(), wakeUpTimesMillis, @@ -348,43 +349,81 @@ private static Optional getNextAwakeInSleepPeriod(final List getAwakeCluster(final List clusters, final VotingSegment awake){ + for(final Segment cluster:clusters){ + if(cluster.getStartTimestamp() >= awake.getStartTimestamp() && cluster.getEndTimestamp() <= awake.getEndTimestamp()){ + return Optional.of(cluster); + } + } + return Optional.absent(); + } + protected static Pair votingSafeGuardPickWakeUp(final List awakesUnfiltered, + final List clusterAmplitudeData, final SleepPeriod sleepPeriod, final Map> featuresNotCapped, final Pair wakeUpTimesSafeGuarded, final long wakeUpMillisPredicted){ + + final List clusters = MotionCluster.toSegments(clusterAmplitudeData); if(wakeUpMillisPredicted < wakeUpTimesSafeGuarded.getFirst()){ return wakeUpTimesSafeGuarded; } - final Optional lastAwakeInSleepPeriod = getNextAwakeInSleepPeriod(awakesUnfiltered, sleepPeriod, wakeUpTimesSafeGuarded.getFirst()); - if(!lastAwakeInSleepPeriod.isPresent()){ + final Optional nextAwakeInSleepPeriod = getNextAwakeInSleepPeriod(awakesUnfiltered, sleepPeriod, wakeUpTimesSafeGuarded.getFirst()); + if(!nextAwakeInSleepPeriod.isPresent()){ return wakeUpTimesSafeGuarded; } - if(lastAwakeInSleepPeriod.get().getStartTimestamp() - wakeUpTimesSafeGuarded.getFirst() < 15 * DateTimeConstants.MILLIS_PER_MINUTE){ + final Optional awakeCluster = getAwakeCluster(clusters, nextAwakeInSleepPeriod.get()); + if(!awakeCluster.isPresent()){ + return wakeUpTimesSafeGuarded; + } + + final Optional wakeUpCluster = getClusterByTimeMillis(clusters, wakeUpTimesSafeGuarded.getFirst(), 0, 0); + if(!wakeUpCluster.isPresent()){ + return wakeUpTimesSafeGuarded; + } + + final Optional maxScoreInAwake = getMaxScore(featuresNotCapped, + MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, + awakeCluster.get().getStartTimestamp(), + awakeCluster.get().getEndTimestamp()); + final Optional maxScoreWakeUp = getMaxScore(featuresNotCapped, + MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, + wakeUpCluster.get().getStartTimestamp(), + wakeUpCluster.get().getEndTimestamp()); + if(maxScoreInAwake.isPresent() && maxScoreWakeUp.isPresent()){ + if(maxScoreInAwake.get().amplitude * 5 < maxScoreWakeUp.get().amplitude){ + // The max score is not in that awake + return wakeUpTimesSafeGuarded; + } + } + + if(nextAwakeInSleepPeriod.get().getStartTimestamp() - wakeUpTimesSafeGuarded.getFirst() < 15 * DateTimeConstants.MILLIS_PER_MINUTE){ LOGGER.debug("HAPPY USER3: Last wake up close to safeguarded result, last wake up {} - {}", - new DateTime(lastAwakeInSleepPeriod.get().getStartTimestamp(), - DateTimeZone.forOffsetMillis(lastAwakeInSleepPeriod.get().getOffsetMillis())), - new DateTime(lastAwakeInSleepPeriod.get().getEndTimestamp(), - DateTimeZone.forOffsetMillis(lastAwakeInSleepPeriod.get().getOffsetMillis()))); + new DateTime(nextAwakeInSleepPeriod.get().getStartTimestamp(), + DateTimeZone.forOffsetMillis(nextAwakeInSleepPeriod.get().getOffsetMillis())), + new DateTime(nextAwakeInSleepPeriod.get().getEndTimestamp(), + DateTimeZone.forOffsetMillis(nextAwakeInSleepPeriod.get().getOffsetMillis()))); return wakeUpTimesSafeGuarded; } final Optional maxScore = getMaxScore(featuresNotCapped, MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, - lastAwakeInSleepPeriod.get().getStartTimestamp(), - lastAwakeInSleepPeriod.get().getEndTimestamp()); + awakeCluster.get().getStartTimestamp(), + awakeCluster.get().getEndTimestamp()); LOGGER.debug("Wrong safeguarding: Last wake up far away from safeguarded result, last wake up {} - {}", - new DateTime(lastAwakeInSleepPeriod.get().getStartTimestamp(), - DateTimeZone.forOffsetMillis(lastAwakeInSleepPeriod.get().getOffsetMillis())), - new DateTime(lastAwakeInSleepPeriod.get().getEndTimestamp(), - DateTimeZone.forOffsetMillis(lastAwakeInSleepPeriod.get().getOffsetMillis()))); + new DateTime(nextAwakeInSleepPeriod.get().getStartTimestamp(), + DateTimeZone.forOffsetMillis(nextAwakeInSleepPeriod.get().getOffsetMillis())), + new DateTime(nextAwakeInSleepPeriod.get().getEndTimestamp(), + DateTimeZone.forOffsetMillis(nextAwakeInSleepPeriod.get().getOffsetMillis()))); if(!maxScore.isPresent()){ - return new Pair<>(lastAwakeInSleepPeriod.get().getStartTimestamp(), lastAwakeInSleepPeriod.get().getEndTimestamp()); + return new Pair<>(awakeCluster.get().getStartTimestamp(), awakeCluster.get().getEndTimestamp()); } - return new Pair<>(maxScore.get().timestamp, lastAwakeInSleepPeriod.get().getEndTimestamp()); + return new Pair<>(maxScore.get().timestamp, + awakeCluster.get().getEndTimestamp()); } protected static Pair safeGuardPickWakeUp(final List clusters, @@ -505,7 +544,8 @@ protected static Pair safeGuardPickWakeUp(final List maxWakeUpScoreOptional = getMaxScore(featuresNotCapped, MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, lastSegmentInSleepPeriod.getEndTimestamp() - 60 * DateTimeConstants.MILLIS_PER_MINUTE, - lastSegmentInSleepPeriod.getEndTimestamp() ); + lastSegmentInSleepPeriod.getEndTimestamp()); + // No obvious motion, edge case if(!maxWakeUpScoreOptional.isPresent()){ return new Pair<>(lastSegmentInSleepPeriod.getStartTimestamp(), lastSegmentInSleepPeriod.getEndTimestamp()); } From 251094fce98af549bd01af95c0b7c6cfcf3e7b52 Mon Sep 17 00:00:00 2001 From: Pang Wu Date: Fri, 17 Apr 2015 15:26:23 -0700 Subject: [PATCH 5/5] Revert --- .../hello/suripu/algorithm/sleep/Vote.java | 74 ------------------- 1 file changed, 74 deletions(-) diff --git a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java index 302aa4c64..e86d81a05 100644 --- a/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java +++ b/suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java @@ -282,12 +282,6 @@ private SleepEvents aggregate(final SleepEvents defaultEvents) sleepPeriod, getAggregatedFeatures(), defaultEvents.wakeUp.getStartTimestamp()); - wakeUpTimesMillis = votingSafeGuardPickWakeUp(this.sleepPeriod.getAwakePeriods(false), - clusterCopy, - sleepPeriod, - getAggregatedFeatures(), - wakeUpTimesMillis, - defaultEvents.wakeUp.getStartTimestamp()); long wakeUpMillis = findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getFirst()); long outBedMillis = findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getSecond()); @@ -358,74 +352,6 @@ protected static Optional getAwakeCluster(final List clusters, return Optional.absent(); } - protected static Pair votingSafeGuardPickWakeUp(final List awakesUnfiltered, - final List clusterAmplitudeData, - final SleepPeriod sleepPeriod, - final Map> featuresNotCapped, - final Pair wakeUpTimesSafeGuarded, - final long wakeUpMillisPredicted){ - - final List clusters = MotionCluster.toSegments(clusterAmplitudeData); - if(wakeUpMillisPredicted < wakeUpTimesSafeGuarded.getFirst()){ - return wakeUpTimesSafeGuarded; - } - - final Optional nextAwakeInSleepPeriod = getNextAwakeInSleepPeriod(awakesUnfiltered, sleepPeriod, wakeUpTimesSafeGuarded.getFirst()); - if(!nextAwakeInSleepPeriod.isPresent()){ - return wakeUpTimesSafeGuarded; - } - - final Optional awakeCluster = getAwakeCluster(clusters, nextAwakeInSleepPeriod.get()); - if(!awakeCluster.isPresent()){ - return wakeUpTimesSafeGuarded; - } - - final Optional wakeUpCluster = getClusterByTimeMillis(clusters, wakeUpTimesSafeGuarded.getFirst(), 0, 0); - if(!wakeUpCluster.isPresent()){ - return wakeUpTimesSafeGuarded; - } - - final Optional maxScoreInAwake = getMaxScore(featuresNotCapped, - MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, - awakeCluster.get().getStartTimestamp(), - awakeCluster.get().getEndTimestamp()); - final Optional maxScoreWakeUp = getMaxScore(featuresNotCapped, - MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, - wakeUpCluster.get().getStartTimestamp(), - wakeUpCluster.get().getEndTimestamp()); - if(maxScoreInAwake.isPresent() && maxScoreWakeUp.isPresent()){ - if(maxScoreInAwake.get().amplitude * 5 < maxScoreWakeUp.get().amplitude){ - // The max score is not in that awake - return wakeUpTimesSafeGuarded; - } - } - - if(nextAwakeInSleepPeriod.get().getStartTimestamp() - wakeUpTimesSafeGuarded.getFirst() < 15 * DateTimeConstants.MILLIS_PER_MINUTE){ - LOGGER.debug("HAPPY USER3: Last wake up close to safeguarded result, last wake up {} - {}", - new DateTime(nextAwakeInSleepPeriod.get().getStartTimestamp(), - DateTimeZone.forOffsetMillis(nextAwakeInSleepPeriod.get().getOffsetMillis())), - new DateTime(nextAwakeInSleepPeriod.get().getEndTimestamp(), - DateTimeZone.forOffsetMillis(nextAwakeInSleepPeriod.get().getOffsetMillis()))); - return wakeUpTimesSafeGuarded; - } - final Optional maxScore = getMaxScore(featuresNotCapped, - MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE, - awakeCluster.get().getStartTimestamp(), - awakeCluster.get().getEndTimestamp()); - - LOGGER.debug("Wrong safeguarding: Last wake up far away from safeguarded result, last wake up {} - {}", - new DateTime(nextAwakeInSleepPeriod.get().getStartTimestamp(), - DateTimeZone.forOffsetMillis(nextAwakeInSleepPeriod.get().getOffsetMillis())), - new DateTime(nextAwakeInSleepPeriod.get().getEndTimestamp(), - DateTimeZone.forOffsetMillis(nextAwakeInSleepPeriod.get().getOffsetMillis()))); - - if(!maxScore.isPresent()){ - return new Pair<>(awakeCluster.get().getStartTimestamp(), awakeCluster.get().getEndTimestamp()); - } - return new Pair<>(maxScore.get().timestamp, - awakeCluster.get().getEndTimestamp()); - } - protected static Pair safeGuardPickWakeUp(final List clusters, final SleepPeriod sleepPeriod, final Map> featuresNotCapped,