diff --git a/README.md b/README.md index 2ea75bc8..d24eeff3 100644 --- a/README.md +++ b/README.md @@ -136,10 +136,6 @@ next_allowed_tokens = guide.advance(allowed_tokens[-1]) # To check if Guide is finished: guide.is_finished() -# If it's finished then this assertion holds: -assert guide.get_tokens() == [vocabulary.get_eos_token_id()] -``` - ## How to contribute? ### Setup diff --git a/src/index.rs b/src/index.rs index 04ca41ca..7f75208f 100644 --- a/src/index.rs +++ b/src/index.rs @@ -226,9 +226,6 @@ impl Index { /// Returns transition state for a given state and token id or `None` otherwise. pub fn next_state(&self, state: &StateId, token_id: &TokenId) -> Option { - if token_id == &self.eos_token_id { - return None; - } Some(*self.transitions.get(state)?.get(token_id)?) } @@ -284,7 +281,7 @@ mod tests { assert_eq!(index.next_state(&initial_state, token_id), Some(state)); assert!(index.is_final_state(&state)); - assert_eq!(index.next_state(&state, &eos_token_id), None); + assert_eq!(index.next_state(&state, &eos_token_id), Some(state)); assert_eq!(index.next_state(&state, token_id), None); } diff --git a/src/json_schema/parsing.rs b/src/json_schema/parsing.rs index 7ed23f41..7f2f7a97 100644 --- a/src/json_schema/parsing.rs +++ b/src/json_schema/parsing.rs @@ -65,7 +65,7 @@ impl<'a> Parser<'a> { fn parse_empty_object(&mut self) -> Result { // JSON Schema Spec: Empty object means unconstrained, any json type is legal - let types = vec![ + let types = [ json!({"type": "boolean"}), json!({"type": "null"}), json!({"type": "number"}), diff --git a/tests/test_guide.py b/tests/test_guide.py index 36e7b95b..9c8f50f9 100644 --- a/tests/test_guide.py +++ b/tests/test_guide.py @@ -210,7 +210,7 @@ def test_rollback_interface(index): ([1], True), # single allowed token: accept ([2], True), # different allowed token: accept ([1, 1], False), # too long for r"[1-9]": reject - ([2, 3], False), # extra token: reject + ([2, 3], True), # extra token: accept at the final state ], ) def test_accepts_tokens_correctness(index, seq, expected):