Skip to content

Refactor and search to minute level #789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pangw wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Refactor and search to minute level #789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ef07ca8
Fix first cluster search
Apr 16, 2015
cf2910d
WIP
Apr 16, 2015
15c11d6
check point
Apr 17, 2015
3da3c0b
WIP
Apr 17, 2015
251094f
Revert
Apr 17, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/SleepPeriod.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,22 @@ private boolean isAwake(final long startMillis, final long endMillis){
return false;
}

public List<Segment> getAwakePeriods(final boolean debug){
private double getVote(final long startMillis, final long endMillis){

double maxVote = 0;
for(int i = 0; i < this.votes.size(); i++){
final Pair<Long, Double> vote = this.votes.get(i);
if(vote.getFirst() >= startMillis && vote.getFirst() <= endMillis && vote.getSecond() > maxVote){
maxVote = vote.getSecond();
}
}
return maxVote;
}

public List<VotingSegment> getAwakePeriods(final boolean debug){
long startMillis = 0;
long endMillis = 0;
final List<Segment> result = new ArrayList<>();
final List<VotingSegment> result = new ArrayList<>();

for(int i = 0; i < this.votes.size(); i++){
final Pair<Long, Double> vote = this.votes.get(i);
Expand All @@ -97,7 +109,7 @@ public List<Segment> getAwakePeriods(final boolean debug){
endMillis = vote.getFirst();
}else {
if(isAwake(startMillis, endMillis)){
result.add(new Segment(startMillis, endMillis, this.getOffsetMillis()));
result.add(new VotingSegment(startMillis, endMillis, this.getOffsetMillis(), getVote(startMillis, endMillis)));
if(debug){
LOGGER.debug("User awake at {} - {}",
new DateTime(startMillis, DateTimeZone.forOffsetMillis(this.getOffsetMillis())),
Expand All @@ -110,7 +122,7 @@ public List<Segment> getAwakePeriods(final boolean debug){
}

if(isAwake(startMillis, endMillis)){
result.add(new Segment(startMillis, endMillis, this.getOffsetMillis()));
result.add(new VotingSegment(startMillis, endMillis, this.getOffsetMillis(), getVote(startMillis, endMillis)));
if(debug){
LOGGER.debug("User awake at {} - {}",
new DateTime(startMillis, DateTimeZone.forOffsetMillis(this.getOffsetMillis())),
Expand Down
128 changes: 96 additions & 32 deletions suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/Vote.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public class Vote {
private final MotionScoreAlgorithm motionScoreAlgorithmDefault;

private final MotionCluster motionCluster;
private final List<AmplitudeData> alignedAmplitude;
private final Map<MotionFeatures.FeatureType, List<AmplitudeData>> aggregatedFeatures;
private final double rawAmpMean;
private final double rawKickOffMean;
Expand Down Expand Up @@ -62,6 +63,7 @@ public Vote(final List<AmplitudeData> rawData,
this.rawKickOffMean = NumericalUtils.mean(noDuplicateKickOffCounts);

List<AmplitudeData> dataWithGapFilled = DataUtils.fillMissingValues(noDuplicates, DateTimeConstants.MILLIS_PER_MINUTE);
this.alignedAmplitude = Lists.newArrayList(dataWithGapFilled);
List<AmplitudeData> alignedKickOffs = DataUtils.fillMissingValues(noDuplicateKickOffCounts, DateTimeConstants.MILLIS_PER_MINUTE);
if(insertEmpty) {
final int insertLengthMin = 20;
Expand Down Expand Up @@ -116,22 +118,26 @@ public Vote(final List<AmplitudeData> rawData,


public final List<Segment> getAwakes(final long fallAsleepMillis, final long wakeUpMillis, final boolean debug){
final List<Segment> allAwakesPeriods = this.sleepPeriod.getAwakePeriods(debug);
final List<Segment> awakesInTheRange = new ArrayList<>();
final List<VotingSegment> allAwakesPeriods = this.sleepPeriod.getAwakePeriods(debug);
final List<VotingSegment> awakesInTheRange = new ArrayList<>();

for(final Segment segment:allAwakesPeriods){
for(final VotingSegment segment:allAwakesPeriods){
if(fallAsleepMillis <= segment.getStartTimestamp() && wakeUpMillis >= segment.getEndTimestamp()){
awakesInTheRange.add(segment);
}
}
final List<Segment> smoothedAwakes = smoothAwakes(awakesInTheRange, MotionCluster.toSegments(this.motionCluster.getCopyOfClusters()));
return smoothedAwakes;
final List<VotingSegment> smoothedAwakes = smoothAwakes(awakesInTheRange, MotionCluster.toSegments(this.motionCluster.getCopyOfClusters()));
final List<Segment> results = new ArrayList<>();
for(final VotingSegment votingSegment:smoothedAwakes){
results.add(votingSegment);
}
return results;
}


private List<Segment> getAwakesInTimeSpanMillis(final List<Segment> awakes, final long startMillis, final long endMillis){
final List<Segment> result = new ArrayList<>();
for(final Segment awake:awakes){
private List<VotingSegment> getAwakesInTimeSpanMillis(final List<VotingSegment> awakes, final long startMillis, final long endMillis){
final List<VotingSegment> result = new ArrayList<>();
for(final VotingSegment awake:awakes){
if(awake.getStartTimestamp() >= startMillis && awake.getEndTimestamp() <= endMillis){
result.add(awake);
}
Expand All @@ -140,14 +146,14 @@ private List<Segment> getAwakesInTimeSpanMillis(final List<Segment> awakes, fina
return result;
}

private List<Segment> filterAwakeFragments(final List<Segment> awakesInMotionCluster){
final List<Segment> result = new ArrayList<>();
private List<VotingSegment> filterAwakeFragments(final List<VotingSegment> awakesInMotionCluster){
final List<VotingSegment> result = new ArrayList<>();
if(awakesInMotionCluster.size() < 2){
return awakesInMotionCluster;
}

for(final Segment awake:awakesInMotionCluster){
if(awake.getDuration() < 20 * DateTimeConstants.MILLIS_PER_MINUTE){
for(final VotingSegment awake:awakesInMotionCluster){
if(awake.getDuration() < 10 * DateTimeConstants.MILLIS_PER_MINUTE){
continue;
}
result.add(awake);
Expand All @@ -160,10 +166,10 @@ private List<Segment> filterAwakeFragments(final List<Segment> awakesInMotionClu
* Idea: if multiple awakes is in the same motion cluster, filter out those less than
* 20 minutes, less is more.
*/
private List<Segment> smoothAwakes(final List<Segment> awakes, final List<Segment> motionClusters){
final List<Segment> smoothedAwakes = new ArrayList<>();
private List<VotingSegment> smoothAwakes(final List<VotingSegment> awakes, final List<Segment> motionClusters){
final List<VotingSegment> smoothedAwakes = new ArrayList<>();
for(final Segment currentCluster:motionClusters){
final List<Segment> awakesInMotionCluster = getAwakesInTimeSpanMillis(awakes,
final List<VotingSegment> awakesInMotionCluster = getAwakesInTimeSpanMillis(awakes,
currentCluster.getStartTimestamp(),
currentCluster.getEndTimestamp());
smoothedAwakes.addAll(filterAwakeFragments(awakesInMotionCluster));
Expand Down Expand Up @@ -258,11 +264,10 @@ private SleepEvents<Segment> aggregate(final SleepEvents<Segment> defaultEvents)
clusterCopy,
this.getAggregatedFeatures(),
defaultEvents.fallAsleep.getStartTimestamp());
long sleepTimeMillis = sleepTimesMillis.getSecond();
inBed = new Segment(sleepTimesMillis.getFirst(), sleepTimesMillis.getFirst() + DateTimeConstants.MILLIS_PER_MINUTE, sleep.getOffsetMillis());
sleep = new Segment(sleepTimeMillis,
sleepTimeMillis + DateTimeConstants.MILLIS_PER_MINUTE,
defaultEvents.fallAsleep.getOffsetMillis());
long sleepTimeMillis = findNearestDataTime(this.alignedAmplitude, sleepTimesMillis.getSecond());
long inBedTimeMillis = findNearestDataTime(this.alignedAmplitude, sleepTimesMillis.getFirst());
inBed = new Segment(inBedTimeMillis, inBedTimeMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.goToBed.getOffsetMillis());
sleep = new Segment(sleepTimeMillis, sleepTimeMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.fallAsleep.getOffsetMillis());
if(inBed.getStartTimestamp() > sleep.getStartTimestamp()){
inBed = new Segment(sleep.getStartTimestamp() - 10 * DateTimeConstants.MILLIS_PER_MINUTE,
sleep.getStartTimestamp() - 9 * DateTimeConstants.MILLIS_PER_MINUTE,
Expand All @@ -273,17 +278,42 @@ private SleepEvents<Segment> aggregate(final SleepEvents<Segment> defaultEvents)
Segment wakeUp = defaultEvents.wakeUp;
Segment outBed = defaultEvents.outOfBed;

final Pair<Long, Long> wakeUpTimesMillis = safeGuardPickWakeUp(clusterCopy,
Pair<Long, Long> wakeUpTimesMillis = safeGuardPickWakeUp(clusterCopy,
sleepPeriod,
getAggregatedFeatures(),
defaultEvents.wakeUp.getStartTimestamp());
wakeUp = new Segment(wakeUpTimesMillis.getFirst(), wakeUpTimesMillis.getFirst() + DateTimeConstants.MILLIS_PER_MINUTE, wakeUp.getOffsetMillis());
outBed = new Segment(wakeUpTimesMillis.getSecond(), wakeUpTimesMillis.getSecond() + DateTimeConstants.MILLIS_PER_MINUTE, inBed.getOffsetMillis());

long wakeUpMillis = findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getFirst());
long outBedMillis = findNearestDataTime(this.alignedAmplitude, wakeUpTimesMillis.getSecond());
wakeUp = new Segment(wakeUpMillis, wakeUpMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.wakeUp.getOffsetMillis());
outBed = new Segment(outBedMillis, outBedMillis + DateTimeConstants.MILLIS_PER_MINUTE, defaultEvents.outOfBed.getOffsetMillis());


return SleepEvents.create(inBed, sleep, wakeUp, outBed);
}

private long findNearestDataTime(final List<AmplitudeData> data, final long targetMillis){
int minDiff = Integer.MAX_VALUE;
long time = 0;
for(int i = 0; i < data.size(); i++){
if(data.get(i).amplitude == 0){
continue;
}

final int diff = (int) Math.abs(data.get(i).timestamp - targetMillis);
if(diff < minDiff){
minDiff = diff;
time = data.get(i).timestamp;
}
}

if(minDiff == Integer.MAX_VALUE){
return targetMillis;
}

return time;
}


private static Pair<Long, Long> predictionBoundsMillis(final long wakeUpMillisPredicted, final Optional<Segment> predictionSegment){
if(!predictionSegment.isPresent()){
Expand All @@ -292,6 +322,36 @@ private static Pair<Long, Long> predictionBoundsMillis(final long wakeUpMillisPr
return new Pair<>(wakeUpMillisPredicted, predictionSegment.get().getEndTimestamp());
}

private static Optional<VotingSegment> getNextAwakeInSleepPeriod(final List<VotingSegment> awakesUnfiltered,
final SleepPeriod sleepPeriod,
final long wakeUpSafeGuarded){
for(final VotingSegment votingSegment:awakesUnfiltered){
if(votingSegment.getStartTimestamp() > sleepPeriod.getEndTimestamp()){
continue;
}

if(votingSegment.getDuration() < 20 * DateTimeConstants.MILLIS_PER_MINUTE){
continue;
}

if(votingSegment.getStartTimestamp() >= wakeUpSafeGuarded ||
(votingSegment.getStartTimestamp() < wakeUpSafeGuarded && votingSegment.getEndTimestamp() >= wakeUpSafeGuarded)){
return Optional.of(votingSegment);
}
}

return Optional.absent();
}

protected static Optional<Segment> getAwakeCluster(final List<Segment> clusters, final VotingSegment awake){
for(final Segment cluster:clusters){
if(cluster.getStartTimestamp() >= awake.getStartTimestamp() && cluster.getEndTimestamp() <= awake.getEndTimestamp()){
return Optional.of(cluster);
}
}
return Optional.absent();
}

protected static Pair<Long, Long> safeGuardPickWakeUp(final List<ClusterAmplitudeData> clusters,
final SleepPeriod sleepPeriod,
final Map<MotionFeatures.FeatureType, List<AmplitudeData>> featuresNotCapped,
Expand All @@ -313,8 +373,10 @@ protected static Pair<Long, Long> safeGuardPickWakeUp(final List<ClusterAmplitud
return new Pair<>(wakeUpMillisPredicted, lastSegmentInSleepPeriod.getEndTimestamp());
}

// predict < last cluster
if(wakeUpMillisPredicted < lastSegmentInSleepPeriod.getStartTimestamp()) {
// predict << last segment of sleep period
// predict << last segment of sleep period, this user toss and turn a lot during sleep
// The prediction can landed on one of the heavy toss-and-turn cluster.
if (lastSegmentInSleepPeriod.getStartTimestamp() - wakeUpMillisPredicted > 40 * DateTimeConstants.MILLIS_PER_MINUTE) {
LOGGER.debug("Predicted too far way from end, predicted {}",
new DateTime(wakeUpMillisPredicted, DateTimeZone.forOffsetMillis(lastSegment.getOffsetMillis())));
Expand All @@ -327,8 +389,8 @@ protected static Pair<Long, Long> safeGuardPickWakeUp(final List<ClusterAmplitud
MotionFeatures.FeatureType.DENSITY_DROP_BACKTRACK_MAX_AMPLITUDE,
wakeUpMillisPredicted,
lastSegmentInSleepPeriod.getEndTimestamp());

if(maxSleepScoreOptional.isPresent()){
if(maxSleepScoreOptional.isPresent()){ // deal with edge case, noo significant motion.
// decide if we should safe guard this result to the max score end
if(wakeUpMillisPredicted <= maxSleepScoreOptional.get().timestamp &&
maxSleepScoreOptional.get().timestamp < lastSegmentInSleepPeriod.getStartTimestamp()){
LOGGER.debug("Max drop between prediction and last segment detected, prediction is likely right.");
Expand All @@ -346,7 +408,8 @@ protected static Pair<Long, Long> safeGuardPickWakeUp(final List<ClusterAmplitud
}
return new Pair<>(maxWakeUpScoreOptional.get().timestamp, clusters.get(maxScoreCluster.getSecond()).timestamp);
}else {

// Prediction is nearby the last cluster, but still not the last
// fallback to the moment has max wake up score.
LOGGER.debug("OK USER: Predict not too far from end");
final Optional<AmplitudeData> maxWakeUpScoreOptional = getMaxScore(featuresNotCapped,
MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE,
Expand All @@ -365,11 +428,11 @@ protected static Pair<Long, Long> safeGuardPickWakeUp(final List<ClusterAmplitud
}
}

// predict > last segment in sleep period
// predict > last segment in sleep period, possible caused by maid or wrong sleep period detection
if(lastSegment.getStartTimestamp() == lastSegmentInSleepPeriod.getStartTimestamp() &&
lastSegment.getEndTimestamp() == lastSegmentInSleepPeriod.getEndTimestamp()){

// No motion cluster after end of sleep period.
// No motion cluster after end of sleep period. error not caused by maid motion
LOGGER.debug("-------------* No maid found, last motion cluster {} - {}",
new DateTime(lastSegment.getStartTimestamp(), DateTimeZone.forOffsetMillis(lastSegment.getOffsetMillis())),
new DateTime(lastSegment.getEndTimestamp(), DateTimeZone.forOffsetMillis(lastSegment.getOffsetMillis())));
Expand All @@ -390,7 +453,7 @@ protected static Pair<Long, Long> safeGuardPickWakeUp(final List<ClusterAmplitud
return new Pair<>(wakeUpMillisPredicted, lastMotionMillis);

}else {

// Maid or partner motion found, maid might cause multiple motion clusters
// last segment in sleep period < last segment && predict > last segment in sleep period
LOGGER.debug("-------------* Maid found, last motion cluster in sleep period {} - {}",
new DateTime(lastSegmentInSleepPeriod.getStartTimestamp(), DateTimeZone.forOffsetMillis(lastSegmentInSleepPeriod.getOffsetMillis())),
Expand All @@ -403,11 +466,12 @@ protected static Pair<Long, Long> safeGuardPickWakeUp(final List<ClusterAmplitud
return predictionBoundsMillis(wakeUpMillisPredicted, predictionSegment);
}


// Fallback to the last segment inside sleep period
final Optional<AmplitudeData> maxWakeUpScoreOptional = getMaxScore(featuresNotCapped,
MotionFeatures.FeatureType.DENSITY_BACKWARD_AVERAGE_AMPLITUDE,
lastSegmentInSleepPeriod.getEndTimestamp() - 60 * DateTimeConstants.MILLIS_PER_MINUTE,
lastSegmentInSleepPeriod.getEndTimestamp() );
lastSegmentInSleepPeriod.getEndTimestamp());
// No obvious motion, edge case
if(!maxWakeUpScoreOptional.isPresent()){
return new Pair<>(lastSegmentInSleepPeriod.getStartTimestamp(), lastSegmentInSleepPeriod.getEndTimestamp());
}
Expand Down
14 changes: 14 additions & 0 deletions suripu-algorithm/src/main/java/com/hello/suripu/algorithm/sleep/VotingSegment.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package com.hello.suripu.algorithm.sleep;

import com.hello.suripu.algorithm.core.Segment;

/**
* Created by pangwu on 4/14/15.
*/
public class VotingSegment extends Segment {
public final double vote;
public VotingSegment(final long startTimestampMillis, final long endTimestampMillis, final int offsetMillis, final double vote){
super(startTimestampMillis, endTimestampMillis, offsetMillis);
this.vote = vote;
}
}