diff --git a/tests/test_model.py b/tests/test_model.py new file mode 100644 index 0000000..e6e8f8c --- /dev/null +++ b/tests/test_model.py @@ -0,0 +1,6 @@ +import tiktoken +def test_gpt5_encoding(): + encoding = tiktoken.encoding_for_model("gpt-5") + assert encoding.name == "o200k_base" + tokens = encoding.encode("hello world") + assert tokens == [24912, 2375] # Verify o200k_base behavior \ No newline at end of file diff --git a/tiktoken/model.py b/tiktoken/model.py index 0f9c011..5c669af 100644 --- a/tiktoken/model.py +++ b/tiktoken/model.py @@ -32,6 +32,7 @@ "o3": "o200k_base", "o4-mini": "o200k_base", # chat + "gpt-5": "o200k_base", "gpt-4.1": "o200k_base", "gpt-4o": "o200k_base", "gpt-4": "cl100k_base",