From a74183ddb4ab8383bfe214b3745eb8a0a99ee47a Mon Sep 17 00:00:00 2001 From: Determinant Date: Thu, 25 Jun 2015 12:56:45 +0800 Subject: let HTK I/O implementation be a single package --- tnet_io/KaldiLib/Tokenizer.cc | 53 ------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 tnet_io/KaldiLib/Tokenizer.cc (limited to 'tnet_io/KaldiLib/Tokenizer.cc') diff --git a/tnet_io/KaldiLib/Tokenizer.cc b/tnet_io/KaldiLib/Tokenizer.cc deleted file mode 100644 index 0c49050..0000000 --- a/tnet_io/KaldiLib/Tokenizer.cc +++ /dev/null @@ -1,53 +0,0 @@ -#include "Tokenizer.h" -#include "string.h" - -namespace TNet -{ - //**************************************************************************** - //**************************************************************************** - void - Tokenizer:: - AddString(const char* pString) - { - // copy into string struct, which is more convenient - std::string aux_string(pString); - std::string aux_record; - std::string::size_type cur_pos = 0; - std::string::size_type old_pos = 0; - std::string::size_type search_start = 0; - - // make sure we have enough space - aux_record.reserve(aux_string.length()); - - // find all of separators and make a list of tokens - while(old_pos < std::string::npos) { - // find the next separator - cur_pos = aux_string.find_first_of(mSeparator, search_start); - - // if backslash is in front of separator, ignore this separator - if (cur_pos != 0 && cur_pos != std::string::npos && - pString[cur_pos - 1] == '\\') { - search_start = cur_pos + 1; - continue; - } - - // we don't want to have empty records - if (!(cur_pos == old_pos && mSkipEmpty)) { - // extract token - aux_record.insert(0, pString+old_pos, cur_pos==std::string::npos ? strlen(pString+old_pos) : cur_pos - old_pos); - // insert to list - this->push_back(aux_record); - - // we don't need the contents of the token - aux_record.erase(); - } - - // update old position so that it points behind the separator - old_pos = cur_pos < std::string::npos ? cur_pos + 1 : cur_pos; - search_start = old_pos; - } - } - - -} // namespace TNet - -- cgit v1.2.3