tools_to_remember/String_tokenizer.h at master · timothyshull/tools_to_remember · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#ifndef STRING_TOKENIZER_H
#define STRING_TOKENIZER_H

#include <string>
#include <unordered_set>

template<typename Char_type>
class String_tokenizer {
public:
    using String_type = typename std::basic_string<Char_type>;
    using Size_type = typename std::basic_string<Char_type>::size_type;
    using Iterator_type = typename std::basic_string<Char_type>::const_iterator;
private:
    std::unordered_set<Char_type> _delimiters;
    String_type _str;
    std::size_t _count;
    Iterator_type _begin;
    Iterator_type _end;

public:
    String_tokenizer() = delete;

    String_tokenizer(const String_tokenizer&) = default;

    String_tokenizer(String_tokenizer&&) = default;

    ~String_tokenizer() = default;

    String_tokenizer& operator=(const String_tokenizer&) = default;

    String_tokenizer& operator=(String_tokenizer&&) = default;

    String_tokenizer(const String_type& s, const String_type& delims = " \f\n\r\t\v\b")
            : _str{s},
              _count{std::numeric_limits<std::size_t>::max()},
              _delimiters{delims.begin(), delims.end()}
    {
        _begin = std::find_if(_str.cbegin(), _str.cend(), [this](const Char_type& a) { return this->_delimiters.find(a) == _delimiters.end(); });
        _end = std::find_if(_begin, _str.cend(), [this](const Char_type& a) { return this->_delimiters.find(a) != _delimiters.end(); });
    }

    std::size_t count_tokens()
    {
        if (_count != std::numeric_limits<std::size_t>::max()) { return _count; }

        _count = 0;

        Iterator_type begin = _str.cbegin();
        // while (begin != _str.cend()) {
        while (true) {
            begin = std::find_if(begin, _str.cend(), [this](const Char_type& a) { return this->_delimiters.find(a) == _delimiters.end(); });
            if (begin == _str.cend()) { break; }
            ++_count;
            begin = std::find_if(begin, _str.cend(), [this](const Char_type& a) { return this->_delimiters.find(a) != _delimiters.end(); });
        }
        return _count;
    }

    inline bool has_next() const noexcept { return _begin != _end; }

    std::basic_string<Char_type> next_token()
    {
        std::basic_string<Char_type> s;
        if (_begin != _str.cend()) { s = {_begin, _end}; }
        if (_end == _str.cend()) { _begin = _end; } // no more tokens
        else {
            _begin = std::find_if(_end, _str.cend(), [this](const Char_type& a) { return this->_delimiters.find(a) == _delimiters.end(); });
            _end = std::find_if(_begin, _str.cend(), [this](const Char_type& a) { return this->_delimiters.find(a) != _delimiters.end(); });
        }
        return s;
    }
};

#endif // STRING_TOKENIZER_H