⚡️ Speed up method Tokenizer.texts_to_matrix by 10%

[codeflash-ai]

📄 10% (0.10x) speedup for Tokenizer.texts_to_matrix in keras/src/legacy/preprocessing/text.py

⏱️ Runtime : 8.51 milliseconds → 7.76 milliseconds (best of 250 runs)

📝 Explanation and details

The optimization achieves a 9% speedup by making two key changes to the inner loop of sequences_to_matrix:

What was optimized:

1. Replaced collections.defaultdict(int) with plain Python dict - The optimized version uses manual if j in counts checks instead of defaultdict's automatic zero initialization
2. Removed unnecessary list() conversion - Changed list(counts.items()) to direct counts.items() iteration
3. Pre-computed sequence length - Stored seq_len = len(seq) once instead of calling len(seq) repeatedly in freq mode

Why this leads to speedup:

• Plain dict with manual checks is faster than defaultdict for dense token indices because it avoids the overhead of the default factory function call on each new key
• Eliminating the list() conversion saves memory allocation and copying overhead
• Pre-computing sequence length reduces repeated function calls in freq mode

Performance impact analysis:
The line profiler shows the most significant improvements in the token counting loop:

• Dictionary operations (counts[j] += 1 equivalent) improved from 2.05ms to 1.33ms + 1.15ms = 2.48ms total, but with better cache locality
• The counts.items() iteration improved from 2.08ms to 1.88ms due to avoiding list conversion

Test case performance:
The optimization is particularly effective for:

• Large vocabularies (19.8-19.9% faster on 1000-word vocab tests)
• Character-level tokenization (12.9-15.7% faster)
• Dense token distributions where most dictionary lookups are hits rather than misses

The optimization maintains identical output behavior while reducing overhead in the critical token-counting bottleneck that processes every token in every sequence.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 80 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

import collections
import warnings

import numpy as np
# imports
import pytest  # used for our unit tests
from keras.src.legacy.preprocessing.text import Tokenizer

# unit tests

# ----------- BASIC TEST CASES -----------

def test_binary_mode_basic():
    # Test simple binary mode conversion
    tokenizer = Tokenizer(num_words=5)
    # Simulate fitting
    tokenizer.word_index = {"hello": 1, "world": 2, "test": 3}
    texts = ["hello world", "test"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="binary"); matrix = codeflash_output # 20.3μs -> 17.9μs (13.4% faster)

def test_count_mode_basic():
    # Test count mode conversion
    tokenizer = Tokenizer(num_words=5)
    tokenizer.word_index = {"foo": 1, "bar": 2}
    texts = ["foo foo bar", "bar"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="count"); matrix = codeflash_output # 19.4μs -> 18.1μs (6.78% faster)

def test_freq_mode_basic():
    # Test freq mode conversion
    tokenizer = Tokenizer(num_words=4)
    tokenizer.word_index = {"a": 1, "b": 2}
    texts = ["a b b", "a a"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="freq"); matrix = codeflash_output # 19.0μs -> 17.8μs (7.10% faster)

def test_oov_token_basic():
    # Test OOV token handling
    tokenizer = Tokenizer(num_words=4, oov_token="<OOV>")
    tokenizer.word_index = {"foo": 1, "<OOV>": 2}
    texts = ["foo bar"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="binary"); matrix = codeflash_output # 14.2μs -> 13.3μs (6.74% faster)

def test_char_level_basic():
    # Test character-level tokenization
    tokenizer = Tokenizer(num_words=5, char_level=True)
    tokenizer.word_index = {"a": 1, "b": 2}
    texts = ["ab", "ba"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="binary"); matrix = codeflash_output # 10.9μs -> 9.41μs (15.7% faster)

def test_custom_analyzer_basic():
    # Test with a custom analyzer function
    def analyzer(text):
        return text.split("-")
    tokenizer = Tokenizer(num_words=4, analyzer=analyzer)
    tokenizer.word_index = {"foo": 1, "bar": 2}
    texts = ["foo-bar"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="binary"); matrix = codeflash_output # 8.91μs -> 7.83μs (13.8% faster)

# ----------- EDGE TEST CASES -----------

def test_empty_texts():
    # Test with empty text list
    tokenizer = Tokenizer(num_words=3)
    tokenizer.word_index = {"a": 1}
    codeflash_output = tokenizer.texts_to_matrix([], mode="binary"); matrix = codeflash_output # 4.15μs -> 4.07μs (1.89% faster)

def test_empty_string():
    # Test with empty string in texts
    tokenizer = Tokenizer(num_words=3)
    tokenizer.word_index = {"a": 1}
    codeflash_output = tokenizer.texts_to_matrix([""], mode="binary"); matrix = codeflash_output # 10.9μs -> 10.9μs (0.156% faster)

def test_all_oov():
    # All words are OOV
    tokenizer = Tokenizer(num_words=4, oov_token="<OOV>")
    tokenizer.word_index = {"<OOV>": 1}
    texts = ["foo bar baz"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="binary"); matrix = codeflash_output # 16.2μs -> 15.0μs (7.66% faster)

def test_num_words_limit():
    # Test num_words limit truncates indices
    tokenizer = Tokenizer(num_words=2)
    tokenizer.word_index = {"a": 1, "b": 2}
    texts = ["a b"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="binary"); matrix = codeflash_output # 15.3μs -> 13.7μs (11.4% faster)

def test_unknown_mode_raises():
    # Unknown mode should raise ValueError
    tokenizer = Tokenizer(num_words=3)
    tokenizer.word_index = {"a": 1}
    texts = ["a"]
    with pytest.raises(ValueError):
        tokenizer.texts_to_matrix(texts, mode="unknown_mode") # 13.5μs -> 12.2μs (10.3% faster)

def test_no_num_words_and_no_word_index():
    # Should raise ValueError if neither num_words nor word_index set
    tokenizer = Tokenizer()
    texts = ["a"]
    with pytest.raises(ValueError):
        tokenizer.texts_to_matrix(texts, mode="binary") # 9.33μs -> 9.08μs (2.72% faster)

def test_tfidf_requires_document_count():
    # Should raise ValueError if document_count not set for tfidf
    tokenizer = Tokenizer(num_words=3)
    tokenizer.word_index = {"a": 1}
    texts = ["a"]
    with pytest.raises(ValueError):
        tokenizer.texts_to_matrix(texts, mode="tfidf") # 9.41μs -> 9.19μs (2.38% faster)

def test_tfidf_basic():
    # Test tfidf calculation
    tokenizer = Tokenizer(num_words=3)
    tokenizer.word_index = {"a": 1, "b": 2}
    tokenizer.document_count = 2
    tokenizer.index_docs = {1: 1, 2: 1}
    texts = ["a b a"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="tfidf"); matrix = codeflash_output # 31.8μs -> 30.3μs (4.95% faster)
    # tf = 1 + log(count), idf = log(1 + doc_count/(1+index_docs))
    tf_a = 1 + np.log(2)
    idf_a = np.log(1 + 2/(1+1))
    tf_b = 1 + np.log(1)
    idf_b = np.log(1 + 2/(1+1))

def test_texts_as_lists():
    # Test texts as lists of tokens
    tokenizer = Tokenizer(num_words=4)
    tokenizer.word_index = {"foo": 1, "bar": 2}
    texts = [["foo", "bar", "foo"], ["bar"]]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="count"); matrix = codeflash_output # 12.4μs -> 11.1μs (12.1% faster)

def test_split_and_filters():
    # Test custom split and filters
    tokenizer = Tokenizer(num_words=4, split="|", filters="!")
    tokenizer.word_index = {"foo": 1, "bar": 2}
    texts = ["foo|bar!foo"]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="count"); matrix = codeflash_output # 12.2μs -> 11.5μs (6.21% faster)

# ----------- LARGE SCALE TEST CASES -----------

def test_large_num_words():
    # Test with large num_words and many word_index entries
    num_words = 1000
    tokenizer = Tokenizer(num_words=num_words)
    # Create word_index mapping for 1..999
    tokenizer.word_index = {f"word{i}": i for i in range(1, num_words)}
    # Create a text with every word once
    texts = [" ".join([f"word{i}" for i in range(1, num_words)])]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="binary"); matrix = codeflash_output # 395μs -> 330μs (19.8% faster)
    # All indices 1..999 should be set to 1
    for i in range(1, num_words):
        pass

def test_large_batch():
    # Test with many texts
    num_words = 10
    tokenizer = Tokenizer(num_words=num_words)
    tokenizer.word_index = {f"w{i}": i for i in range(1, num_words)}
    texts = [f"w{i} w{(i+1)%num_words}" for i in range(1, 100)]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="binary"); matrix = codeflash_output # 336μs -> 311μs (8.13% faster)
    # Check that each row has two 1s
    for i, row in enumerate(matrix):
        pass

def test_large_oov():
    # Test large batch with OOV tokens
    num_words = 20
    tokenizer = Tokenizer(num_words=num_words, oov_token="<OOV>")
    tokenizer.word_index = {f"w{i}": i for i in range(1, num_words)}
    tokenizer.word_index["<OOV>"] = num_words-1
    texts = [f"w{i} unknown{i}" for i in range(1, 100)]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="binary"); matrix = codeflash_output # 359μs -> 339μs (5.75% faster)
    # Each row should have one in OOV index and one in word index
    for row in matrix:
        pass

def test_large_freq():
    # Test freq mode with large batch
    num_words = 50
    tokenizer = Tokenizer(num_words=num_words)
    tokenizer.word_index = {f"w{i}": i for i in range(1, num_words)}
    texts = [f"w{i} w{i} w{(i+1)%num_words}" for i in range(1, 100)]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="freq"); matrix = codeflash_output # 372μs -> 343μs (8.43% faster)
    for i, row in enumerate(matrix):
        pass

def test_large_tfidf():
    # Test tfidf mode with large batch
    num_words = 30
    tokenizer = Tokenizer(num_words=num_words)
    tokenizer.word_index = {f"w{i}": i for i in range(1, num_words)}
    tokenizer.document_count = 100
    tokenizer.index_docs = {i: 10 for i in range(1, num_words)}
    texts = [f"w{i} w{i}" for i in range(1, 30)]
    codeflash_output = tokenizer.texts_to_matrix(texts, mode="tfidf"); matrix = codeflash_output # 158μs -> 149μs (5.94% faster)
    for i, row in enumerate(matrix):
        # Only one word present per row
        idx = i+1
        # All others should be zero
        for j in range(num_words):
            if j != idx:
                pass
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

import collections
import warnings

import numpy as np
# imports
import pytest  # used for our unit tests
from keras.src.legacy.preprocessing.text import Tokenizer

# =========================
# Unit Tests for Tokenizer.texts_to_matrix
# =========================

# ----------- Basic Test Cases -----------

def setup_basic_tokenizer():
    # Helper to create a tokenizer with a fixed vocabulary
    t = Tokenizer(num_words=6)
    # Simulate fitting on texts
    t.word_index = {'the': 1, 'cat': 2, 'sat': 3, 'on': 4, 'mat': 5}
    t.index_word = {v: k for k, v in t.word_index.items()}
    return t

def test_binary_mode_basic():
    # Test binary mode with simple sentences
    t = setup_basic_tokenizer()
    texts = ["The cat sat", "On the mat"]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 21.7μs -> 20.3μs (7.05% faster)
    # All other positions should be 0
    for i in range(6):
        if i not in [1,2,3]:
            pass
        if i not in [1,4,5]:
            pass

def test_count_mode_basic():
    # Test count mode with repeated words
    t = setup_basic_tokenizer()
    texts = ["cat cat sat", "mat mat mat"]
    codeflash_output = t.texts_to_matrix(texts, mode="count"); mat = codeflash_output # 19.7μs -> 18.8μs (4.87% faster)
    # All other positions should be 0
    for i in range(6):
        if i not in [2,3]:
            pass
        if i != 5:
            pass

def test_freq_mode_basic():
    # Test freq mode with repeated words
    t = setup_basic_tokenizer()
    texts = ["cat cat sat"]
    codeflash_output = t.texts_to_matrix(texts, mode="freq"); mat = codeflash_output # 14.8μs -> 13.7μs (7.62% faster)
    for i in range(6):
        if i not in [2,3]:
            pass

def test_oov_token_basic():
    # Test OOV token handling
    t = Tokenizer(num_words=6, oov_token="<OOV>")
    t.word_index = {'the': 1, 'cat': 2, 'sat': 3, 'on': 4, 'mat': 5, "<OOV>": 0}
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = ["dog sat"]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 14.5μs -> 13.7μs (5.98% faster)
    for i in range(6):
        if i not in [0,3]:
            pass

def test_char_level_basic():
    # Test char_level tokenization
    t = Tokenizer(num_words=5, char_level=True)
    t.word_index = {'a':1, 'b':2, 'c':3, 'd':4}
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = ["abac", "dabc"]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 12.2μs -> 10.9μs (12.9% faster)

def test_custom_split_and_filters():
    # Test custom split and filters
    t = Tokenizer(num_words=5, split="-", filters="!")
    t.word_index = {'hello':1, 'world':2}
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = ["hello-world!"]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 12.0μs -> 10.7μs (12.3% faster)
    for i in range(5):
        if i not in [1,2]:
            pass

# ----------- Edge Test Cases -----------

def test_empty_texts():
    # Test with empty list of texts
    t = setup_basic_tokenizer()
    codeflash_output = t.texts_to_matrix([], mode="binary"); mat = codeflash_output # 4.30μs -> 4.08μs (5.42% faster)

def test_empty_string():
    # Test with empty string in texts
    t = setup_basic_tokenizer()
    codeflash_output = t.texts_to_matrix([""], mode="binary"); mat = codeflash_output # 10.7μs -> 10.7μs (0.065% slower)

def test_unknown_mode_raises():
    # Test unknown mode raises ValueError
    t = setup_basic_tokenizer()
    with pytest.raises(ValueError):
        t.texts_to_matrix(["the cat"], mode="unknown_mode") # 14.4μs -> 13.6μs (5.67% faster)

def test_no_num_words_and_no_word_index():
    # Test error when num_words is None and word_index is empty
    t = Tokenizer()
    with pytest.raises(ValueError):
        t.texts_to_matrix(["hello world"], mode="binary") # 10.2μs -> 10.2μs (0.029% slower)

def test_num_words_limits_vocab():
    # Test that num_words limits matrix columns
    t = Tokenizer(num_words=3)
    t.word_index = {'a':1, 'b':2, 'c':3, 'd':4}
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = ["a b c d"]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 17.7μs -> 16.1μs (9.81% faster)

def test_tfidf_mode_requires_document_count():
    # Test tfidf mode raises ValueError if document_count is not set
    t = setup_basic_tokenizer()
    with pytest.raises(ValueError):
        t.texts_to_matrix(["the cat"], mode="tfidf") # 10.1μs -> 9.64μs (4.32% faster)

def test_tfidf_mode_computation():
    # Test tfidf mode with known document_count and index_docs
    t = setup_basic_tokenizer()
    t.document_count = 2
    t.index_docs = collections.defaultdict(int, {1:2, 2:1, 3:1, 4:1, 5:1})
    texts = ["the cat sat"]
    codeflash_output = t.texts_to_matrix(texts, mode="tfidf"); mat = codeflash_output # 33.2μs -> 31.4μs (5.51% faster)
    for idx in [1,2,3]:
        pass
    for idx in [0,4,5]:
        pass

def test_texts_as_lists():
    # Test texts as lists of tokens
    t = Tokenizer(num_words=5)
    t.word_index = {'a':1, 'b':2, 'c':3}
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = [["A", "B", "C"], ["b", "c"]]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 12.8μs -> 11.2μs (14.2% faster)

def test_analyzer_function():
    # Test custom analyzer function
    t = Tokenizer(num_words=4, analyzer=lambda x: x.split("-"))
    t.word_index = {'foo':1, 'bar':2, 'baz':3}
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = ["foo-bar-baz"]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 9.66μs -> 8.47μs (14.1% faster)

# ----------- Large Scale Test Cases -----------

def test_large_vocabulary_and_texts():
    # Test with large vocabulary and texts
    vocab_size = 1000
    t = Tokenizer(num_words=vocab_size)
    # Simulate fitting: word_index = {'word0':1, ..., 'word999':999}
    t.word_index = {f'word{i}':i for i in range(1, vocab_size)}
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = [" ".join([f"word{i}" for i in range(1, vocab_size)])]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 392μs -> 326μs (19.9% faster)
    # All columns except 0 should be 1
    for i in range(1, vocab_size):
        pass

def test_large_batch_of_texts():
    # Test with large batch of texts
    t = Tokenizer(num_words=10)
    t.word_index = {f'w{i}':i for i in range(1,10)}
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = [f"w{i} w{(i+1)%9+1}" for i in range(1, 1000)]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 3.32ms -> 3.06ms (8.53% faster)
    # Each row should have two 1s
    for row in range(999):
        tokens = texts[row].split()
        indices = [t.word_index.get(tok) for tok in tokens]
        for idx in indices:
            pass
        for i in range(10):
            if i not in indices:
                pass

def test_large_oov_handling():
    # Test with large OOV token handling
    t = Tokenizer(num_words=10, oov_token="<OOV>")
    t.word_index = {f'w{i}':i for i in range(1,9)}
    t.word_index["<OOV>"] = 0
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = [f"unknown{i} w{(i%8)+1}" for i in range(100)]
    codeflash_output = t.texts_to_matrix(texts, mode="binary"); mat = codeflash_output # 386μs -> 355μs (8.81% faster)
    # Each row should have OOV and one in-vocab token
    for row in range(100):
        idx = t.word_index.get(f'w{(row%8)+1}')
        for i in range(10):
            if i not in [0, idx]:
                pass

def test_large_freq_mode():
    # Test freq mode with large batch
    t = Tokenizer(num_words=10)
    t.word_index = {f'w{i}':i for i in range(1,10)}
    t.index_word = {v: k for k, v in t.word_index.items()}
    texts = [f"w1 w2 w2 w3 w3 w3" for _ in range(500)]
    codeflash_output = t.texts_to_matrix(texts, mode="freq"); mat = codeflash_output # 2.34ms -> 2.13ms (10.2% faster)
    # Each row: w1:1/6, w2:2/6, w3:3/6
    for row in range(500):
        for i in range(10):
            if i not in [1,2,3]:
                pass
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

To edit these changes git checkout codeflash/optimize-Tokenizer.texts_to_matrix-mirhcvsl and push.