From a74183ddb4ab8383bfe214b3745eb8a0a99ee47a Mon Sep 17 00:00:00 2001
From: Determinant <ted.sybil@gmail.com>
Date: Thu, 25 Jun 2015 12:56:45 +0800
Subject: let HTK I/O implementation be a single package

---
 htk_io/src/KaldiLib/Tokenizer.h | 45 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 htk_io/src/KaldiLib/Tokenizer.h

(limited to 'htk_io/src/KaldiLib/Tokenizer.h')

diff --git a/htk_io/src/KaldiLib/Tokenizer.h b/htk_io/src/KaldiLib/Tokenizer.h
new file mode 100644
index 0000000..1be717b
--- /dev/null
+++ b/htk_io/src/KaldiLib/Tokenizer.h
@@ -0,0 +1,45 @@
+#include <list>
+#include <string>
+
+namespace TNet {
+  /** 
+   * @brief General string tokenizer
+   */
+  class Tokenizer 
+  : public std::list<std::string>
+  {
+  public:
+    // Constructors and Destructors ............................................
+    Tokenizer(const char* pSeparator, bool skipEmpty = false)
+    : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty)
+    {}
+
+    Tokenizer(const char* pString, const char* pSeparator, bool skipEmpty = false)
+    : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty)
+    { AddString(pString); }
+
+    ~Tokenizer()
+    {}
+
+    /** 
+     * @brief Parses a string and appends the tokens to the list
+     * @param pString string to parse
+     */
+    void
+    AddString(const char* pString);
+
+    /** 
+     * @brief Constant accessor to the separators string
+     * @return Const refference
+     */
+    const std::string&
+    Separator() const
+    {return mSeparator;}
+
+  private:
+    std::string mSeparator;   ///< holds the list of separators
+    bool        mSkipEmpty;   ///< if true, multiple separators will be regarded as one
+  }; // class Tokenizer
+} // namespace TNet
+
+
-- 
cgit v1.2.3-70-g09d2