Repair Interpolation

.gitignore

@@ -9,4 +9,6 @@
 .idea/
 target/
 
-lambda$*.json
+lambda$*.json
+
+.DS_Store

src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java

@@ -262,7 +262,7 @@ public String update(Integer id, String json, boolean autoCommit) throws SQLExce
      * false for interpolation.
      */
     public int normalizeStopTimesForPattern(int id, int beginWithSequence) throws SQLException {
-        return normalizeStopTimesForPattern(id, beginWithSequence, false);
+        return normalizeStopTimesForPattern(id, beginWithSequence, false, false);
     }
 
     /**
@@ -271,7 +271,7 @@ public int normalizeStopTimesForPattern(int id, int beginWithSequence) throws SQ
      *
      * @return number of stop times updated
      */
-    public int normalizeStopTimesForPattern(int id, int beginWithSequence, boolean interpolateStopTimes) throws SQLException {
+    public int normalizeStopTimesForPattern(int id, int beginWithSequence, boolean interpolateStopTimes, boolean ignoreNonBlankStopTimes) throws SQLException {
         try {
             JDBCTableReader<PatternStop> patternStops = new JDBCTableReader(
                 Table.PATTERN_STOP,
@@ -288,7 +288,7 @@ public int normalizeStopTimesForPattern(int id, int beginWithSequence, boolean i
                     patternStopsToNormalize.add(patternStop);
                 }
             }
-            int stopTimesUpdated = updateStopTimesForPatternStops(patternStopsToNormalize, interpolateStopTimes);
+            int stopTimesUpdated = updateStopTimesForPatternStops(patternStopsToNormalize, interpolateStopTimes, ignoreNonBlankStopTimes);
             connection.commit();
             return stopTimesUpdated;
         } catch (Exception e) {
@@ -788,27 +788,46 @@ private int interpolateTimesFromTimepoints(
         PatternStop patternStop,
         List<PatternStop> timepoints,
         Integer timepointNumber,
-        double previousShapeDistTraveled
+        PatternStop prevPatternStop
     ) {
         if (timepointNumber == 0 || timepoints.size() == 1 || timepointNumber >= timepoints.size()) {
             throw new IllegalStateException("Issue in pattern stops which prevents interpolation (e.g. less than 2 timepoints)");
         }
         PatternStop nextTimepoint = timepoints.get(timepointNumber);
-        PatternStop lastTimepoint = timepoints.get(timepointNumber-1);
+        PatternStop prevTimepoint = timepoints.get(timepointNumber-1);
 
         if (
             nextTimepoint == null ||
             nextTimepoint.default_travel_time == Entity.INT_MISSING ||
             nextTimepoint.shape_dist_traveled == Entity.DOUBLE_MISSING ||
-            lastTimepoint.shape_dist_traveled == Entity.DOUBLE_MISSING
+            prevTimepoint.shape_dist_traveled == Entity.DOUBLE_MISSING ||
+            (prevPatternStop != null && prevPatternStop.shape_dist_traveled == Entity.DOUBLE_MISSING)
         ) {
             throw new IllegalStateException("Error with stop time interpolation: timepoint or shape_dist_traveled is null");
         }
 
-        double timepointSpeed = (nextTimepoint.shape_dist_traveled - lastTimepoint.shape_dist_traveled) / nextTimepoint.default_travel_time;
-        return (int) Math.round((patternStop.shape_dist_traveled - previousShapeDistTraveled) / timepointSpeed);
+        double prevSdt = prevPatternStop == null ? 0 : prevPatternStop.shape_dist_traveled;
+
+        double timepointSpeed = (nextTimepoint.shape_dist_traveled - prevTimepoint.shape_dist_traveled) / nextTimepoint.default_travel_time;
+        return (int) Math.round((patternStop.shape_dist_traveled - prevSdt) / timepointSpeed);
     }
 
+    private boolean checkIfArrivalTimeExists(int stopSequence, String trip_id) throws SQLException {
+        String sql = String.format(
+                "select st.trip_id, arrival_time from %s.stop_times st where stop_sequence = ? " +
+                        "and st.trip_id = ?",
+                tablePrefix
+        );
+        PreparedStatement statement = connection.prepareStatement(sql);
+        statement.setInt(1, stopSequence);
+        statement.setString(2, trip_id);
+        ResultSet resultSet = statement.executeQuery();
+        while (resultSet.next()) {
+            int arrivalTime = resultSet.getInt(2);
+            if (arrivalTime <= 0) return false;
+        }
+        return true;
+    }
     /**
      * Normalizes all stop times' arrivals and departures for an ordered set of pattern stops. This set can be the full
      * set of stops for a pattern or just a subset. Typical usage for this method would be to overwrite the arrival and
@@ -818,8 +837,9 @@ private int interpolateTimesFromTimepoints(
      * @throws SQLException
      *
      * TODO? add param Set<String> serviceIdFilters service_id values to filter trips on
+     * TODO: Simply logic, avoid use of continue
      */
-    private int updateStopTimesForPatternStops(List<PatternStop> patternStops, boolean interpolateStopTimes) throws SQLException {
+    private int updateStopTimesForPatternStops(List<PatternStop> patternStops, boolean interpolateStopTimes, boolean ignoreNonBlankStopTimes) throws SQLException {
         PatternStop firstPatternStop = patternStops.iterator().next();
         List<PatternStop> timepoints = patternStops.stream().filter(ps -> ps.timepoint == 1).collect(Collectors.toList());
         int firstStopSequence = firstPatternStop.stop_sequence;
@@ -854,43 +874,61 @@ private int updateStopTimesForPatternStops(List<PatternStop> patternStops, boole
         for (String tripId : timesForTripIds.keySet()) {
             // Initialize travel time with previous stop time value.
             int cumulativeTravelTime = timesForTripIds.get(tripId);
-            int cumulativeInterpolatedTime = cumulativeTravelTime;
+            int timepointCumulativeTravelTime = cumulativeTravelTime;
             int timepointNumber = 0;
-            double previousShapeDistTraveled = 0; // Used for calculating timepoint speed for interpolation
+            PatternStop prevPatternStop = null;
             for (PatternStop patternStop : patternStops) {
                 boolean isTimepoint = patternStop.timepoint == 1;
                 if (isTimepoint) timepointNumber++;
+                boolean hasStopTime = ignoreNonBlankStopTimes ? checkIfArrivalTimeExists(patternStop.stop_sequence, tripId) : true;
                 // Gather travel/dwell time for pattern stop (being sure to check for missing values).
                 int travelTime = patternStop.default_travel_time == Entity.INT_MISSING ? 0 : patternStop.default_travel_time;
-                if (interpolateStopTimes) {
+                if (interpolateStopTimes || ignoreNonBlankStopTimes) {
                     if (patternStop.shape_dist_traveled == Entity.DOUBLE_MISSING) {
                         throw new IllegalStateException("Shape_dist_traveled must be defined for all stops in order to perform interpolation");
                     }
                     // Override travel time if we're interpolating between timepoints.
-                    if (!isTimepoint) travelTime = interpolateTimesFromTimepoints(patternStop, timepoints, timepointNumber, previousShapeDistTraveled);
-                    previousShapeDistTraveled += patternStop.shape_dist_traveled;
+                    if (!isTimepoint && !ignoreNonBlankStopTimes) travelTime = interpolateTimesFromTimepoints(patternStop, timepoints, timepointNumber, prevPatternStop);
+
+                    if (ignoreNonBlankStopTimes && !hasStopTime &&patternStop.stop_sequence != 0) travelTime = interpolateTimesFromTimepoints(patternStop, timepoints, timepointNumber, prevPatternStop);
                 }
-                int dwellTime = patternStop.default_dwell_time == Entity.INT_MISSING ? 0 : patternStop.default_dwell_time;
-                int oneBasedIndex = 1;
-                // Increase travel time by current pattern stop's travel and dwell times (and set values for update).
-                if (!isTimepoint && interpolateStopTimes) {
-                    // We don't want to increment the true cumulative travel time because that adjusts the timepoint
-                    // times later in the pattern.
-                    // Dwell times are ignored right now as they do not fit the typical use case for interpolation.
-                    // They may be incorporated by accounting for all dwell times in intermediate stops when calculating
-                    // the timepoint speed.
-                    cumulativeInterpolatedTime += travelTime;
-                    updateStopTimeStatement.setInt(oneBasedIndex++, cumulativeInterpolatedTime);
-                    updateStopTimeStatement.setInt(oneBasedIndex++, cumulativeInterpolatedTime);
-                } else {
-                    cumulativeTravelTime += travelTime;
-                    updateStopTimeStatement.setInt(oneBasedIndex++, cumulativeTravelTime);
-                    cumulativeTravelTime += dwellTime;
-                    updateStopTimeStatement.setInt(oneBasedIndex++, cumulativeTravelTime);
+                int dwellTime = patternStop.default_dwell_time == Entity.INT_MISSING || (interpolateStopTimes && !isTimepoint) ? 0 : patternStop.default_dwell_time;
+
+                // In non-blank stop time mode, only update if our stoptime is blank
+                if (ignoreNonBlankStopTimes) {
+                    if (hasStopTime) {
+                        cumulativeTravelTime = cumulativeTravelTime + travelTime + dwellTime;
+                        prevPatternStop = patternStop;
+                        continue;
+                    }
+
+                    updateStopTimeStatement.setInt(1, cumulativeTravelTime + travelTime);
+                    updateStopTimeStatement.setInt(2, cumulativeTravelTime + travelTime);
+                    updateStopTimeStatement.setString(3, tripId);
+                    updateStopTimeStatement.setInt(4, patternStop.stop_sequence);
+                    stopTimesTracker.addBatch();
                 }
-                updateStopTimeStatement.setString(oneBasedIndex++, tripId);
-                updateStopTimeStatement.setInt(oneBasedIndex++, patternStop.stop_sequence);
+
+
+                // In interpolation mode, don't write changes to db for timepoints
+                // We don't write the travel time, since it is interpolated
+                if (interpolateStopTimes && isTimepoint) {
+                    timepointCumulativeTravelTime = timepointCumulativeTravelTime + travelTime + dwellTime;
+                    // Reset the cumulativeTravelTime to the schedule truth
+                    cumulativeTravelTime = timepointCumulativeTravelTime;
+                    prevPatternStop = patternStop;
+                    continue;
+                }
+                // Increase travel time by current pattern stop's travel and dwell times (and set values for update).
+                cumulativeTravelTime += travelTime;
+                updateStopTimeStatement.setInt(1, cumulativeTravelTime);
+                cumulativeTravelTime += dwellTime;
+                updateStopTimeStatement.setInt(2, cumulativeTravelTime);
+                updateStopTimeStatement.setString(3, tripId);
+                updateStopTimeStatement.setInt(4, patternStop.stop_sequence);
                 stopTimesTracker.addBatch();
+
+                prevPatternStop = patternStop;
             }
         }
         return stopTimesTracker.executeRemaining();
@@ -1621,7 +1659,7 @@ private String[] parseExceptionListField(int id, String namespace, Table table,
         return parsedString.replaceAll("[{}]", "").split("[,]", 0);
     }
 
-    private String getResultSetString(int column, ResultSet resultSet) throws java.sql.SQLException {
+    private String getResultSetString(int column, ResultSet resultSet) throws SQLException {
         String resultSetString = resultSet.getString(column);
         return resultSetString == null ? "" : resultSetString;
     }

src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java

@@ -874,7 +874,7 @@ private static String normalizeStopsForPattern(
         LOG.info("Updated pattern output: {}", updatedPatternOutput);
         // Normalize stop times.
         JdbcTableWriter updateTripWriter = createTestTableWriter(tripsTable);
-        updateTripWriter.normalizeStopTimesForPattern(pattern.id, 0, interpolateStopTimes);
+        updateTripWriter.normalizeStopTimesForPattern(pattern.id, 0, interpolateStopTimes, false);
 
         return createdTrip.trip_id;
     }