summaryrefslogtreecommitdiff
path: root/tnet_io/KaldiLib/Tokenizer.h
diff options
context:
space:
mode:
authorDeterminant <[email protected]>2015-05-29 23:06:58 +0800
committerDeterminant <[email protected]>2015-05-29 23:06:58 +0800
commit74b9f7cb88cd21cfac3c2e50c8efb802485df0c5 (patch)
treebd6e583088a086144acc2d8af3eaca59691194ff /tnet_io/KaldiLib/Tokenizer.h
init
Diffstat (limited to 'tnet_io/KaldiLib/Tokenizer.h')
-rw-r--r--tnet_io/KaldiLib/Tokenizer.h45
1 files changed, 45 insertions, 0 deletions
diff --git a/tnet_io/KaldiLib/Tokenizer.h b/tnet_io/KaldiLib/Tokenizer.h
new file mode 100644
index 0000000..1be717b
--- /dev/null
+++ b/tnet_io/KaldiLib/Tokenizer.h
@@ -0,0 +1,45 @@
+#include <list>
+#include <string>
+
+namespace TNet {
+ /**
+ * @brief General string tokenizer
+ */
+ class Tokenizer
+ : public std::list<std::string>
+ {
+ public:
+ // Constructors and Destructors ............................................
+ Tokenizer(const char* pSeparator, bool skipEmpty = false)
+ : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty)
+ {}
+
+ Tokenizer(const char* pString, const char* pSeparator, bool skipEmpty = false)
+ : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty)
+ { AddString(pString); }
+
+ ~Tokenizer()
+ {}
+
+ /**
+ * @brief Parses a string and appends the tokens to the list
+ * @param pString string to parse
+ */
+ void
+ AddString(const char* pString);
+
+ /**
+ * @brief Constant accessor to the separators string
+ * @return Const refference
+ */
+ const std::string&
+ Separator() const
+ {return mSeparator;}
+
+ private:
+ std::string mSeparator; ///< holds the list of separators
+ bool mSkipEmpty; ///< if true, multiple separators will be regarded as one
+ }; // class Tokenizer
+} // namespace TNet
+
+