htk_io/src/KaldiLib/Tokenizer.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

#include <list>
#include <string>

namespace TNet {
  /** 
   * @brief General string tokenizer
   */
  class Tokenizer 
  : public std::list<std::string>
  {
  public:
    // Constructors and Destructors ............................................
    Tokenizer(const char* pSeparator, bool skipEmpty = false)
    : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty)
    {}

    Tokenizer(const char* pString, const char* pSeparator, bool skipEmpty = false)
    : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty)
    { AddString(pString); }

    ~Tokenizer()
    {}

    /** 
     * @brief Parses a string and appends the tokens to the list
     * @param pString string to parse
     */
    void
    AddString(const char* pString);

    /** 
     * @brief Constant accessor to the separators string
     * @return Const refference
     */
    const std::string&
    Separator() const
    {return mSeparator;}

  private:
    std::string mSeparator;   ///< holds the list of separators
    bool        mSkipEmpty;   ///< if true, multiple separators will be regarded as one
  }; // class Tokenizer
} // namespace TNet