Allow the guide to consume the eos_token

Author: RobinPicard

README.md

@@ -136,10 +136,6 @@ next_allowed_tokens = guide.advance(allowed_tokens[-1])
 # To check if Guide is finished:
 guide.is_finished()
 
-# If it's finished then this assertion holds:
-assert guide.get_tokens() == [vocabulary.get_eos_token_id()]
-```
-
 ## How to contribute?
 
 ### Setup

src/index.rs

@@ -226,9 +226,6 @@ impl Index {
 
     /// Returns transition state for a given state and token id or `None` otherwise.
     pub fn next_state(&self, state: &StateId, token_id: &TokenId) -> Option<StateId> {
-        if token_id == &self.eos_token_id {
-            return None;
-        }
         Some(*self.transitions.get(state)?.get(token_id)?)
     }
 
@@ -284,7 +281,7 @@ mod tests {
         assert_eq!(index.next_state(&initial_state, token_id), Some(state));
         assert!(index.is_final_state(&state));
 
-        assert_eq!(index.next_state(&state, &eos_token_id), None);
+        assert_eq!(index.next_state(&state, &eos_token_id), Some(state));
         assert_eq!(index.next_state(&state, token_id), None);
     }
 

src/json_schema/parsing.rs

@@ -65,7 +65,7 @@ impl<'a> Parser<'a> {
 
     fn parse_empty_object(&mut self) -> Result<String> {
         // JSON Schema Spec: Empty object means unconstrained, any json type is legal
-        let types = vec![
+        let types = [
             json!({"type": "boolean"}),
             json!({"type": "null"}),
             json!({"type": "number"}),

tests/test_guide.py

@@ -210,7 +210,7 @@ def test_rollback_interface(index):
         ([1], True),  # single allowed token: accept
         ([2], True),  # different allowed token: accept
         ([1, 1], False),  # too long for r"[1-9]": reject
-        ([2, 3], False),  # extra token: reject
+        ([2, 3], True),  # extra token: accept at the final state
     ],
 )
 def test_accepts_tokens_correctness(index, seq, expected):