summaryrefslogtreecommitdiff
path: root/htk_io/src/KaldiLib/Tokenizer.h
diff options
context:
space:
mode:
authorDeterminant <[email protected]>2015-06-25 12:56:45 +0800
committerDeterminant <[email protected]>2015-06-25 12:56:45 +0800
commita74183ddb4ab8383bfe214b3745eb8a0a99ee47a (patch)
treed5e69cf8c4c2db2e3a4722778352fc3c95953bb2 /htk_io/src/KaldiLib/Tokenizer.h
parentb6301089cde20f4c825c7f5deaf179082aad63da (diff)
let HTK I/O implementation be a single package
Diffstat (limited to 'htk_io/src/KaldiLib/Tokenizer.h')
-rw-r--r--htk_io/src/KaldiLib/Tokenizer.h45
1 files changed, 45 insertions, 0 deletions
diff --git a/htk_io/src/KaldiLib/Tokenizer.h b/htk_io/src/KaldiLib/Tokenizer.h
new file mode 100644
index 0000000..1be717b
--- /dev/null
+++ b/htk_io/src/KaldiLib/Tokenizer.h
@@ -0,0 +1,45 @@
+#include <list>
+#include <string>
+
+namespace TNet {
+ /**
+ * @brief General string tokenizer
+ */
+ class Tokenizer
+ : public std::list<std::string>
+ {
+ public:
+ // Constructors and Destructors ............................................
+ Tokenizer(const char* pSeparator, bool skipEmpty = false)
+ : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty)
+ {}
+
+ Tokenizer(const char* pString, const char* pSeparator, bool skipEmpty = false)
+ : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty)
+ { AddString(pString); }
+
+ ~Tokenizer()
+ {}
+
+ /**
+ * @brief Parses a string and appends the tokens to the list
+ * @param pString string to parse
+ */
+ void
+ AddString(const char* pString);
+
+ /**
+ * @brief Constant accessor to the separators string
+ * @return Const refference
+ */
+ const std::string&
+ Separator() const
+ {return mSeparator;}
+
+ private:
+ std::string mSeparator; ///< holds the list of separators
+ bool mSkipEmpty; ///< if true, multiple separators will be regarded as one
+ }; // class Tokenizer
+} // namespace TNet
+
+