Improve paging - avoid duplicates and missed itineraries when paging

src/ext/java/org/opentripplanner/ext/transmodelapi/model/PlanResponse.java

@@ -4,7 +4,7 @@
 import java.util.List;
 import org.opentripplanner.api.resource.DebugOutput;
 import org.opentripplanner.model.plan.TripPlan;
-import org.opentripplanner.model.plan.pagecursor.PageCursor;
+import org.opentripplanner.model.plan.paging.cursor.PageCursor;
 import org.opentripplanner.routing.api.response.RoutingError;
 import org.opentripplanner.routing.api.response.TripSearchMetadata;
 

src/ext/java/org/opentripplanner/ext/transmodelapi/model/plan/TripType.java

@@ -12,7 +12,7 @@
 import org.opentripplanner.ext.transmodelapi.model.PlanResponse;
 import org.opentripplanner.ext.transmodelapi.support.GqlUtil;
 import org.opentripplanner.framework.graphql.GraphQLUtils;
-import org.opentripplanner.model.plan.pagecursor.PageCursor;
+import org.opentripplanner.model.plan.paging.cursor.PageCursor;
 
 public class TripType {
 

src/main/java/org/opentripplanner/apis/gtfs/datafetchers/PlanImpl.java

@@ -8,7 +8,7 @@
 import org.opentripplanner.apis.gtfs.generated.GraphQLDataFetchers;
 import org.opentripplanner.model.plan.Itinerary;
 import org.opentripplanner.model.plan.StopArrival;
-import org.opentripplanner.model.plan.pagecursor.PageCursor;
+import org.opentripplanner.model.plan.paging.cursor.PageCursor;
 import org.opentripplanner.routing.api.response.RoutingError;
 import org.opentripplanner.routing.api.response.RoutingResponse;
 import org.opentripplanner.routing.api.response.TripSearchMetadata;

src/main/java/org/opentripplanner/routing/algorithm/filterchain/ListSection.java → src/main/java/org/opentripplanner/framework/collection/ListSection.java

@@ -1,4 +1,4 @@
-package org.opentripplanner.routing.algorithm.filterchain;
+package org.opentripplanner.framework.collection;
 
 /**
  * This enum is used to signal which part of a list an operation apply to. You may remove elements
@@ -9,5 +9,16 @@ public enum ListSection {
   HEAD,
 
   /** The end of the list */
-  TAIL,
+  TAIL;
+
+  public boolean isHead() {
+    return this == HEAD;
+  }
+
+  public ListSection invert() {
+    return switch (this) {
+      case HEAD -> TAIL;
+      case TAIL -> HEAD;
+    };
+  }
 }

src/main/java/org/opentripplanner/framework/collection/ListUtils.java

@@ -10,6 +10,22 @@
 
 public class ListUtils {
 
+  /**
+   * Return the first element in the list. {@code null} is returned if the list is
+   * null or empty.
+   */
+  public static <T> T first(List<T> list) {
+    return list == null || list.isEmpty() ? null : list.getFirst();
+  }
+
+  /**
+   * Return the last element in the list. {@code null} is returned if the list is
+   * null or empty.
+   */
+  public static <T> T last(List<T> list) {
+    return list == null || list.isEmpty() ? null : list.getLast();
+  }
+
   /**
    * Combine a number of collections into a single list.
    */

src/main/java/org/opentripplanner/framework/lang/Box.java

@@ -0,0 +1,61 @@
+package org.opentripplanner.framework.lang;
+
+import java.util.Objects;
+import javax.annotation.Nullable;
+
+/**
+ * A box around a mutable value reference. This can be used inside a lambda or passed into
+ * a function.
+ * @param <T> the type of the wrapped value.
+ */
+public class Box<T> {
+
+  private T value;
+
+  private Box(T value) {
+    this.value = value;
+  }
+
+  public Box() {
+    this(null);
+  }
+
+  public static <T> Box<T> empty() {
+    return new Box<>();
+  }
+
+  public static <T> Box<T> of(T value) {
+    return new Box<>(value);
+  }
+
+  @Nullable
+  public T get() {
+    return value;
+  }
+
+  public void set(@Nullable T value) {
+    this.value = value;
+  }
+
+  public boolean isEmpty() {
+    return value == null;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    Box<?> box = (Box<?>) o;
+    return Objects.equals(value, box.value);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(value);
+  }
+
+  @Override
+  public String toString() {
+    return "[" + value + ']';
+  }
+}

src/main/java/org/opentripplanner/framework/text/CharacterEscapeFormatter.java

@@ -0,0 +1,107 @@
+package org.opentripplanner.framework.text;
+
+/**
+ * This class is used to escape characters in a string, removing a special character from
+ * the string. For example, if you want to make sure a string does not contain {@code ';'},
+ * the {@code ';'} can be replaced with {@code '\+'}. The slash({@code '\'}) is used as an
+ * escape character, so we need to escape all {@code '\'} as well. Now, the escaped string
+ * does not contain the special character anymore. The original string can be computed by
+ * reversing the process.
+ * <p>
+ * A "special-character" is removed from a text using an escape character and
+ * a substitution character. For example, if:
+ * <ul>
+ *   <li>the escape char is '\'</li>
+ *   <li>the special char is ';'</li>
+ *   <li>and the substitution char is '+'</li>
+ * </ul>
+ *
+ * then replace:
+ * <ul>
+ *   <li>'\' with '\\' and</li>
+ *   <li>';' with '\;'</li>
+ * </ul>
+ * To get back the original text, the reverse process using {@link #decode(String)}.
+ * <pre>
+ *
+ * Original: "\tThis;is;an;example\+"
+ * Encoded:  "\\tThis\+is\+an\+example\\+"
+ * Decoded:  "\tThis;is;an;example\+"
+ * </pre>
+ */
+public class CharacterEscapeFormatter {
+
+  private final char escapeChar;
+  private final char specialChar;
+  private final char substitutionChar;
+
+  /**
+   * @param escapeChar the character used as an escape character.
+   * @param specialChar the character to be removed/replaced in the encoded text.
+   * @param substitutionChar the character used together with the escape character to put in the
+   *                         encoded text as a placeholder for the special character.
+   */
+  public CharacterEscapeFormatter(char escapeChar, char specialChar, char substitutionChar) {
+    this.escapeChar = escapeChar;
+    this.specialChar = specialChar;
+    this.substitutionChar = substitutionChar;
+  }
+
+  /**
+   * Encode the given text and replace the {@code specialChar} with a placeholder. The original
+   * text can be retrieved by using {@link #decode(String)}.
+   * @param text the text to encode.
+   * @return the encoded text without the {@code specialChar}.
+   */
+  public String encode(String text) {
+    final var buf = new StringBuilder();
+    for (int i = 0; i < text.length(); ++i) {
+      char ch = text.charAt(i);
+      if (ch == escapeChar) {
+        buf.append(escapeChar).append(escapeChar);
+      } else if (ch == specialChar) {
+        buf.append(escapeChar).append(substitutionChar);
+      } else {
+        buf.append(ch);
+      }
+    }
+    return buf.toString();
+  }
+
+  /**
+   * Return the original text by decoding the encoded text.
+   * @see #encode(String)
+   */
+  public String decode(String encodedText) {
+    if (encodedText.length() < 2) {
+      return encodedText;
+    }
+    final var buf = new StringBuilder();
+    boolean prevEsc = false;
+    for (int i = 0; i < encodedText.length(); ++i) {
+      char ch = encodedText.charAt(i);
+      if (prevEsc) {
+        if (ch == escapeChar) {
+          buf.append(escapeChar);
+        } else if (ch == substitutionChar) {
+          buf.append(specialChar);
+        } else {
+          throw new IllegalStateException(
+            "Unexpected combination of escape-char '%c' and '%c' character at position %d. Text: '%s'.".formatted(
+                escapeChar,
+                ch,
+                i,
+                encodedText
+              )
+          );
+        }
+        prevEsc = false;
+      } else if (ch == escapeChar) {
+        prevEsc = true;
+      } else {
+        buf.append(ch);
+      }
+    }
+    return buf.toString();
+  }
+}

src/main/java/org/opentripplanner/framework/token/Deserializer.java

@@ -1,97 +1,74 @@
 package org.opentripplanner.framework.token;
 
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.time.Duration;
-import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
-import org.opentripplanner.framework.time.DurationUtils;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
 
 class Deserializer {
 
-  private final ByteArrayInputStream input;
+  private static final Pattern SPLIT_PATTERN = Pattern.compile(
+    "[" + TokenFormatterConfiguration.fieldSeparator() + "]"
+  );
+
+  private final List<String> values;
 
   Deserializer(String token) {
-    this.input = new ByteArrayInputStream(Base64.getUrlDecoder().decode(token));
+    byte[] bytes = Base64.getUrlDecoder().decode(token);
+    var tokenFormatter = TokenFormatterConfiguration.tokenFormatter();
+    this.values =
+      Stream.of(SPLIT_PATTERN.split(new String(bytes), -1)).map(tokenFormatter::decode).toList();
   }
 
-  List<Object> deserialize(TokenDefinition definition) throws IOException {
+  List<Object> deserialize(TokenDefinition definition) {
     try {
       // Assume deprecated fields are included in the token
       return readFields(definition, false);
-    } catch (IOException ignore) {
+    } catch (Exception ignore) {
       // If the token is the next version, then deprecated field are removed. Try
       // skipping the deprecated tokens
       return readFields(definition, true);
     }
   }
 
-  private List<Object> readFields(TokenDefinition definition, boolean matchNewVersionPlusOne)
-    throws IOException {
-    input.reset();
+  private List<Object> readFields(TokenDefinition definition, boolean matchNewVersionPlusOne) {
     List<Object> result = new ArrayList<>();
-
-    var in = new ObjectInputStream(input);
-
-    readAndMatchVersion(in, definition, matchNewVersionPlusOne);
+    matchVersion(definition, matchNewVersionPlusOne);
+    int index = 1;
 
     for (FieldDefinition field : definition.listFields()) {
       if (matchNewVersionPlusOne && field.deprecated()) {
         continue;
       }
-      var v = read(in, field);
+      var v = read(field, index);
       if (!field.deprecated()) {
         result.add(v);
       }
+      ++index;
     }
     return result;
   }
 
-  private void readAndMatchVersion(
-    ObjectInputStream in,
-    TokenDefinition definition,
-    boolean matchVersionPlusOne
-  ) throws IOException {
+  private void matchVersion(TokenDefinition definition, boolean matchVersionPlusOne) {
     int matchVersion = (matchVersionPlusOne ? 1 : 0) + definition.version();
 
-    int v = readInt(in);
-    if (v != matchVersion) {
-      throw new IOException(
-        "Version does not match. Token version: " + v + ", schema version: " + definition.version()
+    int version = readVersion();
+    if (version != matchVersion) {
+      throw new IllegalStateException(
+        "Version does not match. Token version: " +
+        version +
+        ", schema version: " +
+        definition.version()
       );
     }
   }
 
-  private Object read(ObjectInputStream in, FieldDefinition field) throws IOException {
-    return switch (field.type()) {
-      case BYTE -> readByte(in);
-      case DURATION -> readDuration(in);
-      case INT -> readInt(in);
-      case STRING -> readString(in);
-      case TIME_INSTANT -> readTimeInstant(in);
-    };
-  }
-
-  private static byte readByte(ObjectInputStream in) throws IOException {
-    return in.readByte();
-  }
-
-  private static int readInt(ObjectInputStream in) throws IOException {
-    return Integer.parseInt(in.readUTF());
-  }
-
-  private static String readString(ObjectInputStream in) throws IOException {
-    return in.readUTF();
-  }
-
-  private static Duration readDuration(ObjectInputStream in) throws IOException {
-    return DurationUtils.duration(in.readUTF());
+  private Object read(FieldDefinition field, int index) {
+    return field.type().stringToValue(values.get(index));
   }
 
-  private static Instant readTimeInstant(ObjectInputStream in) throws IOException {
-    return Instant.parse(in.readUTF());
+  private int readVersion() {
+    return Integer.parseInt(values.get(0));
   }
 }