⚡️ Speed up function text_to_word_sequence by 42%

[codeflash-ai]

📄 42% (0.42x) speedup for text_to_word_sequence in keras/src/legacy/preprocessing/text.py

⏱️ Runtime : 931 microseconds → 655 microseconds (best of 250 runs)

📝 Explanation and details

The optimized code achieves a 42% speedup through three key optimizations:

1. Conditional filter processing: The original code always created a translation dictionary and map even when filters was empty. The optimized version adds an if filters: check, avoiding unnecessary string translation when no filtering is needed.

2. Inline dictionary comprehension: Instead of creating a separate translate_dict variable, the optimized code passes the dictionary comprehension directly to str.maketrans(), eliminating an intermediate object creation.

3. Smart space splitting: When split=" " (the default), the optimized code uses input_text.split() instead of input_text.split(split). Python's parameterless split() is significantly faster as it's optimized for whitespace splitting and automatically handles multiple consecutive spaces without needing the list comprehension filter [i for i in seq if i].

Performance Impact by Test Category:

• Empty filters (e.g., filters=""): Up to 165% faster by completely skipping translation
• Default space splitting: 59-283% faster on large texts by using optimized split() and avoiding list comprehension
• Custom splits: 45-58% faster for non-space delimiters, still benefiting from conditional filtering

Hot Path Relevance: Based on the function references, text_to_word_sequence is called within loops in fit_on_texts() and texts_to_sequences_generator() methods of the Tokenizer class, and in the hashing_trick() function. These are preprocessing steps that process many texts in batches, making this optimization particularly valuable for text preprocessing pipelines where it could significantly reduce training and inference time.

The optimization maintains identical functionality while being most effective for common use cases: default parameters with space-separated text and scenarios with empty filters.

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 62 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

import pytest  # used for our unit tests
from keras.src.legacy.preprocessing.text import text_to_word_sequence

# unit tests

# --- Basic Test Cases ---

def test_basic_simple_sentence():
    # Simple sentence, default filters, lowercasing
    text = "The quick brown fox jumps over the lazy dog."
    expected = [
        "the", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog"
    ]
    codeflash_output = text_to_word_sequence(text) # 8.30μs -> 7.76μs (7.00% faster)

def test_basic_no_punctuation():
    # Sentence without punctuation, default filters, lowercasing
    text = "Hello world this is a test"
    expected = ["hello", "world", "this", "is", "a", "test"]
    codeflash_output = text_to_word_sequence(text) # 7.11μs -> 6.26μs (13.4% faster)

def test_basic_with_numbers():
    # Sentence with numbers, default filters, lowercasing
    text = "I have 2 apples and 10 oranges."
    expected = ["i", "have", "2", "apples", "and", "10", "oranges"]
    codeflash_output = text_to_word_sequence(text) # 7.27μs -> 6.48μs (12.2% faster)

def test_basic_uppercase_and_lower_false():
    # Sentence with uppercase, lower=False
    text = "HELLO World!"
    expected = ["HELLO", "World"]
    codeflash_output = text_to_word_sequence(text, lower=False) # 6.57μs -> 5.79μs (13.5% faster)

def test_basic_custom_split():
    # Sentence with commas, custom split and filters
    text = "one,two,three"
    expected = ["one", "two", "three"]
    codeflash_output = text_to_word_sequence(text, filters="", split=",") # 3.21μs -> 2.04μs (57.4% faster)

def test_basic_custom_filters():
    # Sentence with custom filters
    text = "hello|world|python"
    expected = ["hello", "world", "python"]
    codeflash_output = text_to_word_sequence(text, filters="|") # 3.55μs -> 3.38μs (4.85% faster)

# --- Edge Test Cases ---

def test_edge_empty_string():
    # Empty string should return empty list
    codeflash_output = text_to_word_sequence("") # 6.10μs -> 5.30μs (15.0% faster)

def test_edge_only_punctuation():
    # Only punctuation should return empty list
    codeflash_output = text_to_word_sequence("!!!...,,,") # 6.47μs -> 5.47μs (18.3% faster)

def test_edge_only_spaces():
    # String with only spaces should return empty list
    codeflash_output = text_to_word_sequence("     ") # 6.16μs -> 5.35μs (15.2% faster)

def test_edge_multiple_spaces_between_words():
    # Multiple spaces between words should be handled
    text = "hello     world"
    expected = ["hello", "world"]
    codeflash_output = text_to_word_sequence(text) # 6.80μs -> 5.88μs (15.7% faster)

def test_edge_multiple_split_characters():
    # Custom split, multiple consecutive split chars
    text = "a--b---c"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters="-", split="-") # 3.41μs -> 3.76μs (9.38% slower)

def test_edge_custom_filter_and_split_overlap():
    # Custom filter and split are the same character
    text = "a|b||c|||d"
    expected = ["a", "b", "c", "d"]
    codeflash_output = text_to_word_sequence(text, filters="|", split="|") # 3.44μs -> 3.57μs (3.72% slower)

def test_edge_filter_removes_split_char():
    # If filter removes the split char, splitting should still work
    text = "a b,c"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters=",") # 3.11μs -> 2.71μs (14.6% faster)

def test_edge_unicode_and_non_ascii():
    # Unicode text should be preserved, only ascii filters applied
    text = "naïve café — voilà!"
    expected = ["naïve", "café", "—", "voilà"]
    codeflash_output = text_to_word_sequence(text) # 9.65μs -> 9.01μs (7.13% faster)

def test_edge_tabs_and_newlines():
    # Tabs and newlines are default filters
    text = "hello\tworld\npython"
    expected = ["hello", "world", "python"]
    codeflash_output = text_to_word_sequence(text) # 6.99μs -> 6.50μs (7.44% faster)

def test_edge_mixed_case_with_lower_false():
    # Mixed case, lower=False
    text = "PyThOn TeStInG"
    expected = ["PyThOn", "TeStInG"]
    codeflash_output = text_to_word_sequence(text, lower=False) # 6.59μs -> 6.25μs (5.31% faster)

def test_edge_split_on_non_space():
    # Split on comma, with spaces
    text = "a, b, c"
    expected = ["a", " b", " c"]
    codeflash_output = text_to_word_sequence(text, filters="", split=",") # 3.09μs -> 2.02μs (52.5% faster)

def test_edge_empty_filter():
    # No filters, punctuation remains
    text = "hello, world!"
    expected = ["hello,", "world!"]
    codeflash_output = text_to_word_sequence(text, filters="") # 3.16μs -> 1.25μs (153% faster)

def test_edge_split_is_filter():
    # Split character is also in filters
    text = "a,b,c"
    # If split is ',', and ',' is in filters, all commas are replaced by split, so splitting on ',' still works
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters=",", split=",") # 3.17μs -> 3.60μs (12.0% slower)

def test_edge_multiple_split_and_filter_overlap():
    # Overlapping split and filter, with multiple consecutive
    text = "a,,b,,,c"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters=",", split=",") # 3.35μs -> 3.59μs (6.79% slower)

def test_edge_leading_and_trailing_split():
    # Leading and trailing split characters
    text = ",,a,b,c,,"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters=",", split=",") # 3.30μs -> 3.63μs (9.09% slower)

def test_edge_non_string_input():
    # Non-string input should raise AttributeError
    with pytest.raises(AttributeError):
        text_to_word_sequence(12345) # 1.42μs -> 1.44μs (1.87% slower)

def test_edge_split_is_empty_string():
    # Split cannot be empty string (str.split('') raises ValueError)
    with pytest.raises(ValueError):
        text_to_word_sequence("a b c", split="") # 6.79μs -> 7.42μs (8.43% slower)

# --- Large Scale Test Cases ---

def test_large_long_text():
    # Long text, 1000 words
    text = "word " * 1000
    expected = ["word"] * 1000
    codeflash_output = text_to_word_sequence(text) # 42.4μs -> 26.5μs (59.8% faster)

def test_large_long_text_with_punctuation():
    # Long text with punctuation, 500 sentences
    text = ("Hello, world! " * 500).strip()
    expected = ["hello", "world"] * 500
    codeflash_output = text_to_word_sequence(text) # 55.9μs -> 27.1μs (106% faster)

def test_large_varied_text():
    # Large varied text with numbers, punctuation, and mixed case
    text = " ".join(
        f"Word{i}, value={i*2}!" for i in range(500)
    )
    # Each "Word{i}, value={i*2}!" becomes ["word{i}", "value", "{i*2}"]
    expected = []
    for i in range(500):
        expected.extend([f"word{i}", "value", f"{i*2}"])
    codeflash_output = text_to_word_sequence(text) # 97.5μs -> 61.1μs (59.4% faster)

def test_large_custom_split_and_filter():
    # Large text with custom split and filter
    text = "|".join(f"item{i}" for i in range(1000))
    expected = [f"item{i}" for i in range(1000)]
    codeflash_output = text_to_word_sequence(text, filters="", split="|") # 39.2μs -> 31.9μs (23.1% faster)

def test_large_repeated_punctuation():
    # Large text with repeated punctuation
    text = ("!!!word!!! " * 500).strip()
    expected = ["word"] * 500
    codeflash_output = text_to_word_sequence(text) # 75.6μs -> 19.7μs (283% faster)

def test_large_edge_case_all_filters():
    # String of only filter characters, repeated
    filters = '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n'
    text = filters * 10
    codeflash_output = text_to_word_sequence(text) # 13.5μs -> 6.45μs (109% faster)

def test_large_performance():
    # Large text, check function doesn't hang or crash
    text = " ".join("word" for _ in range(1000))
    expected = ["word"] * 1000
    codeflash_output = text_to_word_sequence(text) # 39.1μs -> 24.6μs (58.9% faster)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

import pytest  # used for our unit tests
from keras.src.legacy.preprocessing.text import text_to_word_sequence

# unit tests

# ------------------------
# 1. BASIC TEST CASES
# ------------------------

def test_basic_simple_sentence():
    # Simple sentence, basic split
    text = "The quick brown fox"
    expected = ["the", "quick", "brown", "fox"]
    codeflash_output = text_to_word_sequence(text) # 6.96μs -> 6.28μs (10.9% faster)

def test_basic_with_punctuation():
    # Sentence with punctuation in default filters
    text = "Hello, world!"
    expected = ["hello", "world"]
    codeflash_output = text_to_word_sequence(text) # 6.66μs -> 5.72μs (16.5% faster)

def test_basic_uppercase_and_lower():
    # Sentence with mixed case, lower=True
    text = "PyThOn Is AwEsOmE"
    expected = ["python", "is", "awesome"]
    codeflash_output = text_to_word_sequence(text) # 6.69μs -> 5.90μs (13.5% faster)

def test_basic_lower_false():
    # lower=False should preserve case
    text = "PyThOn Is AwEsOmE"
    expected = ["PyThOn", "Is", "AwEsOmE"]
    codeflash_output = text_to_word_sequence(text, lower=False) # 6.69μs -> 6.02μs (11.1% faster)

def test_basic_custom_split():
    # Custom split character
    text = "one,two,three"
    expected = ["one", "two", "three"]
    codeflash_output = text_to_word_sequence(text, filters="", split=",") # 3.24μs -> 2.10μs (54.7% faster)

def test_basic_custom_filters():
    # Custom filters removing digits
    text = "abc123 def456"
    expected = ["abc", "def"]
    codeflash_output = text_to_word_sequence(text, filters="123456") # 4.41μs -> 3.92μs (12.6% faster)

def test_basic_multiple_spaces():
    # Multiple spaces should be handled
    text = "a   b    c"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text) # 6.69μs -> 5.98μs (11.9% faster)

# ------------------------
# 2. EDGE TEST CASES
# ------------------------

def test_edge_empty_string():
    # Empty string should return empty list
    codeflash_output = text_to_word_sequence("") # 5.83μs -> 5.32μs (9.66% faster)

def test_edge_only_filters():
    # String with only filter characters
    text = "!!!@@@###"
    expected = []
    codeflash_output = text_to_word_sequence(text) # 6.04μs -> 5.37μs (12.5% faster)

def test_edge_no_filters():
    # No filters, punctuation kept as part of words
    text = "hello,world! foo.bar"
    expected = ["hello,world!", "foo.bar"]
    codeflash_output = text_to_word_sequence(text, filters="") # 3.48μs -> 1.31μs (165% faster)

def test_edge_only_spaces():
    # String with only spaces
    text = "     "
    expected = []
    codeflash_output = text_to_word_sequence(text) # 6.31μs -> 5.47μs (15.4% faster)

def test_edge_leading_trailing_spaces():
    # Leading and trailing spaces should be ignored
    text = "   hello world   "
    expected = ["hello", "world"]
    codeflash_output = text_to_word_sequence(text) # 6.87μs -> 6.00μs (14.5% faster)

def test_edge_consecutive_filters():
    # Multiple consecutive filter chars
    text = "hello!!!world??"
    expected = ["hello", "world"]
    codeflash_output = text_to_word_sequence(text) # 6.75μs -> 6.08μs (10.9% faster)

def test_edge_split_on_filter():
    # Split char is also in filters: should not split on it
    text = "a,b,c"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters="", split=",") # 3.05μs -> 1.97μs (54.7% faster)

def test_edge_split_is_filter():
    # If split is also a filter, filters are replaced with split, so splitting works
    text = "a,b,c"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters=",", split=",") # 3.14μs -> 3.39μs (7.26% slower)

def test_edge_unicode_characters():
    # Unicode characters not in filters should be preserved
    text = "Café naïve déjà-vu"
    expected = ["café", "naïve", "déjà-vu"]
    codeflash_output = text_to_word_sequence(text) # 8.35μs -> 7.89μs (5.80% faster)

def test_edge_unicode_in_filters():
    # Unicode in filters should be removed
    text = "Café☕"
    expected = ["café"]
    codeflash_output = text_to_word_sequence(text, filters='☕') # 4.36μs -> 4.04μs (8.00% faster)

def test_edge_split_on_tab():
    # Split on tab character
    text = "a\tb\tc"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters="", split="\t") # 2.93μs -> 2.01μs (45.9% faster)

def test_edge_split_on_newline():
    # Split on newline character
    text = "a\nb\nc"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters="", split="\n") # 2.82μs -> 1.91μs (47.8% faster)

def test_edge_filter_is_alphanumeric():
    # Filter out letters
    text = "abc def ghi"
    expected = []
    codeflash_output = text_to_word_sequence(text, filters="abcdefghijklmnopqrstuvwxyz ") # 6.77μs -> 6.08μs (11.3% faster)

def test_edge_non_string_input():
    # Non-string input should raise AttributeError
    with pytest.raises(AttributeError):
        text_to_word_sequence(12345) # 1.42μs -> 1.42μs (0.141% slower)

def test_edge_split_is_empty_string():
    # split="" is not allowed by str.split, should raise ValueError
    with pytest.raises(ValueError):
        text_to_word_sequence("a b c", split="") # 7.00μs -> 7.13μs (1.84% slower)

def test_edge_filters_is_empty_string():
    # filters="" should not filter anything
    text = "hello, world!"
    expected = ["hello,", "world!"]
    codeflash_output = text_to_word_sequence(text, filters="") # 3.27μs -> 1.35μs (141% faster)

def test_edge_split_is_multi_char():
    # split is multi-character string, should split as expected
    text = "a--b--c"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters="", split="--") # 3.15μs -> 2.00μs (57.9% faster)

def test_edge_split_and_filter_overlap():
    # split and filters overlap, splitting should still work
    text = "a,b,c"
    expected = ["a", "b", "c"]
    codeflash_output = text_to_word_sequence(text, filters=",", split=",") # 3.16μs -> 3.53μs (10.5% slower)

def test_edge_filter_is_space():
    # filters=" " removes all spaces, so everything is concatenated
    text = "a b c"
    expected = ["abc"]
    codeflash_output = text_to_word_sequence(text, filters=" ", split=" ") # 3.08μs -> 2.80μs (10.2% faster)

# ------------------------
# 3. LARGE SCALE TEST CASES
# ------------------------

def test_large_long_sentence():
    # Very long sentence (repeated pattern)
    text = ("The quick brown fox jumps over the lazy dog. " * 100).strip()
    # Remove punctuation and lowercase
    expected = (["the", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog"] * 100)
    codeflash_output = text_to_word_sequence(text) # 45.7μs -> 29.3μs (56.3% faster)

def test_large_many_unique_words():
    # 1000 unique words
    words = [f"word{i}" for i in range(1000)]
    text = " ".join(words)
    expected = [w.lower() for w in words]
    codeflash_output = text_to_word_sequence(text) # 60.2μs -> 46.2μs (30.2% faster)

def test_large_many_filters():
    # Large number of filter characters
    filters = "".join(chr(i) for i in range(33, 127))  # all printable ASCII except space
    text = "A!B@C#D$E%F^G&H*I(J)K_L+M-N=O{P}Q|R;S:T'U\"V<W>X,Y.Z/?"
    # All filters removed, only letters remain
    expected = [c.lower() for c in "ABCDEFGHIJKLMNOQRSTUVWXYZ"]
    codeflash_output = text_to_word_sequence(text, filters=filters); result = codeflash_output # 12.7μs -> 11.2μs (13.3% faster)
    # Remove empty strings from expected
    expected = [c for c in expected if c]

def test_large_no_filters_long_string():
    # Long string, no filters
    text = "word," * 999 + "word"
    expected = ["word," for _ in range(999)] + ["word"]
    codeflash_output = text_to_word_sequence(text, filters="") # 8.94μs -> 4.69μs (90.6% faster)

def test_large_performance():
    # Large input, check that function completes and output is correct
    text = "a! " * 1000
    expected = ["a"] * 1000
    codeflash_output = text_to_word_sequence(text) # 48.7μs -> 17.0μs (187% faster)

def test_large_unicode():
    # Large input with unicode
    text = ("naïve café déjà-vu! " * 200).strip()
    expected = (["naïve", "café", "déjà-vu"] * 200)
    codeflash_output = text_to_word_sequence(text) # 143μs -> 129μs (10.7% faster)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

To edit these changes git checkout codeflash/optimize-text_to_word_sequence-mirguop5 and push.