diff options
36 files changed, 11451 insertions, 0 deletions
diff --git a/tnet_io/KaldiLib/Common.cc b/tnet_io/KaldiLib/Common.cc new file mode 100644 index 0000000..40909ee --- /dev/null +++ b/tnet_io/KaldiLib/Common.cc @@ -0,0 +1,277 @@ +#include <string> +#include <stdexcept> +#include <cmath> +#include <cfloat> +#include <cstdio> + +#include "Common.h" +#include "MathAux.h" + + +/// Defines the white chars for string trimming +#if !defined(WHITE_CHARS) +# define WHITE_CHARS " \t" +#endif + +namespace TNet { + +#include <ios> + + // Allocating stream variable used by stream modifier MatrixVectorIostreamControl + const int MATRIX_IOS_FORMAT_IWORD = std::ios_base::xalloc(); + + //*************************************************************************** + //*************************************************************************** + int getHTKstr(char *str) + { + char termChar = '\0'; + char *chrptr = str; + + while (std::isspace(*chrptr)) ++chrptr; + + if (*chrptr == '\'' || *chrptr == '"') { + termChar = *chrptr; + chrptr++; + } + + for (; *chrptr; chrptr++) { + if (*chrptr == '\'' || *chrptr == '"') { + if (termChar == *chrptr) { + termChar = '\0'; + chrptr++; + break; + } + } + + if (std::isspace(*chrptr) && !termChar) { + break; + } + + if (*chrptr == '\\') { + ++chrptr; + if (*chrptr == '\0' || (*chrptr >= '0' && *chrptr <= '7' && + (*++chrptr < '0' || *chrptr > '7' || + *++chrptr < '0' || *chrptr > '7'))) { + return -1; + } + + if (*chrptr >= '0' && *chrptr <= '7') { + *chrptr = (char)((*chrptr - '0') + (chrptr[-1] - '0') * 8 + (chrptr[-2] - '0') * 64); + } + } + *str++ = *chrptr; + } + + if (termChar) { + return -2; + } + + *str = '\0'; + + return 0; + } + + + //***************************************************************************** + //***************************************************************************** + void + ParseHTKString(const std::string & rIn, std::string & rOut) + { + int ret_val; + + // the new string will be at most as long as the original, so we allocate + // space + char* new_str = new char[rIn.size() + 1]; + + char* p_htk_str = new_str; + + strcpy(p_htk_str, rIn.c_str()); + ret_val = getHTKstr(p_htk_str); + + // call the function + if (!ret_val) { + rOut = p_htk_str; + } + + delete [] new_str; + + if (ret_val) { + throw std::runtime_error("Error parsing HTK string"); + } + } + + + + //*************************************************************************** + //*************************************************************************** + bool + IsBigEndian() + { + int a = 1; + return (bool) ((char *) &a)[0] != 1; + } + + + //*************************************************************************** + //*************************************************************************** + void + MakeHtkFileName(char* pOutFileName, const char* inFileName, + const char* out_dir, const char* out_ext) + { + const char* base_name; + const char* bname_end = NULL; + const char* chrptr; + + // if (*inFileName == '*' && *++inFileName == '/') ++inFileName; + + // we don't do anything if file is stdin/out + if (!strcmp(inFileName, "-")) + { + pOutFileName[0] = '-'; + pOutFileName[1] = '\0'; + return; + } + + base_name = strrchr(inFileName, '/'); + base_name = base_name != NULL ? base_name + 1 : inFileName; + + if (out_ext) bname_end = strrchr(base_name, '.'); + if (!bname_end) bname_end = base_name + strlen(base_name); + + + if ((chrptr = strstr(inFileName, "/./")) != NULL) + { + // what is in path after /./ serve as base name + base_name = chrptr + 3; + } + /* else if (*inFileName != '/') + { + // if inFileName isn't absolut path, don't forget directory structure + base_name = inFileName; + }*/ + + *pOutFileName = '\0'; + if (out_dir) + { + if (*out_dir) + { + strcat(pOutFileName, out_dir); + strcat(pOutFileName, "/"); + } + strncat(pOutFileName, base_name, bname_end-base_name); + } + else + { + strncat(pOutFileName, inFileName, bname_end-inFileName); + } + + if (out_ext && *out_ext) + { + strcat(pOutFileName, "."); + strcat(pOutFileName, out_ext); + } + } + + + //**************************************************************************** + //**************************************************************************** + bool + CloseEnough(const float f1, const float f2, const float nRounds) + { + bool ret_val = (_ABS((f1 - f2) / (f2 == 0.0f ? 1.0f : f2)) + < (nRounds * FLT_EPSILON)); + + return ret_val; + } + + + //**************************************************************************** + //**************************************************************************** + bool + CloseEnough(const double f1, const double f2, const double nRounds) + { + bool ret_val = (_ABS((f1 - f2) / (f2 == 0.0 ? 1.0 : f2)) + < (nRounds * DBL_EPSILON)); + + return ret_val; + } + + + //**************************************************************************** + //**************************************************************************** + char* + ExpandHtkFilterCmd(const char *command, const char *filename, const char* pFilter) + { + + char *out, *outend; + const char *chrptr = command; + int ndollars = 0; + int fnlen = strlen(filename); + + while (*chrptr++) ndollars += (*chrptr == *pFilter); + + out = (char*) malloc(strlen(command) - ndollars + ndollars * fnlen + 1); + + outend = out; + + for (chrptr = command; *chrptr; chrptr++) { + if (*chrptr == *pFilter) { + strcpy(outend, filename); + outend += fnlen; + } else { + *outend++ = *chrptr; + } + } + *outend = '\0'; + return out; + } + + //*************************************************************************** + //*************************************************************************** + char * + StrToUpper(char *str) + { + char *chptr; + for (chptr = str; *chptr; chptr++) { + *chptr = (char)toupper(*chptr); + } + return str; + } + + + //**************************************************************************** + //**************************************************************************** + std::string& + Trim(std::string& rStr) + { + // WHITE_CHARS is defined in common.h + std::string::size_type pos = rStr.find_last_not_of(WHITE_CHARS); + if(pos != std::string::npos) + { + rStr.erase(pos + 1); + pos = rStr.find_first_not_of(WHITE_CHARS); + if(pos != std::string::npos) rStr.erase(0, pos); + } + else + rStr.erase(rStr.begin(), rStr.end()); + + return rStr; + } + + +} // namespace TNet + +//#ifdef CYGWIN + +void assertf(const char *c, int i, const char *msg){ + printf("Assertion \"%s\" failed: file \"%s\", line %d\n", msg?msg:"(null)", c?c:"(null)", i); + abort(); +} + + +void assertf_throw(const char *c, int i, const char *msg){ + char buf[2000]; + snprintf(buf, 1999, "Assertion \"%s\" failed, throwing exception: file \"%s\", line %d\n", msg?msg:"(null)", c?c:"(null)", i); + throw std::runtime_error((std::string)buf); +} +//#endif diff --git a/tnet_io/KaldiLib/Common.h b/tnet_io/KaldiLib/Common.h new file mode 100644 index 0000000..9cd9658 --- /dev/null +++ b/tnet_io/KaldiLib/Common.h @@ -0,0 +1,233 @@ +#ifndef TNet_Common_h +#define TNet_Common_h + +#include <cstdlib> +#include <string.h> // C string stuff like strcpy +#include <string> +#include <sstream> +#include <stdexcept> + +/* Alignment of critical dynamic data structure + * + * Not all platforms support memalign so we provide a stk_memalign wrapper + * void *stk_memalign( size_t align, size_t size, void **pp_orig ) + * *pp_orig is the pointer that has to be freed afterwards. + */ +#ifdef HAVE_POSIX_MEMALIGN +# define stk_memalign(align,size,pp_orig) \ + ( !posix_memalign( pp_orig, align, size ) ? *(pp_orig) : NULL ) +# ifdef STK_MEMALIGN_MANUAL +# undef STK_MEMALIGN_MANUAL +# endif +#elif defined(HAVE_MEMALIGN) + /* Some systems have memalign() but no declaration for it */ + //void * memalign( size_t align, size_t size ); +# define stk_memalign(align,size,pp_orig) \ + ( *(pp_orig) = memalign( align, size ) ) +# ifdef STK_MEMALIGN_MANUAL +# undef STK_MEMALIGN_MANUAL +# endif +#else /* We don't have any choice but to align manually */ +# define stk_memalign(align,size,pp_orig) \ + (( *(pp_orig) = malloc( size + align - 1 )) ? \ + (void *)( (((unsigned long)*(pp_orig)) + 15) & ~0xFUL ) : NULL ) +# define STK_MEMALIGN_MANUAL +#endif + + +#define swap8(a) { \ + char t=((char*)&a)[0]; ((char*)&a)[0]=((char*)&a)[7]; ((char*)&a)[7]=t;\ + t=((char*)&a)[1]; ((char*)&a)[1]=((char*)&a)[6]; ((char*)&a)[6]=t;\ + t=((char*)&a)[2]; ((char*)&a)[2]=((char*)&a)[5]; ((char*)&a)[5]=t;\ + t=((char*)&a)[3]; ((char*)&a)[3]=((char*)&a)[4]; ((char*)&a)[4]=t;} +#define swap4(a) { \ + char t=((char*)&a)[0]; ((char*)&a)[0]=((char*)&a)[3]; ((char*)&a)[3]=t;\ + t=((char*)&a)[1]; ((char*)&a)[1]=((char*)&a)[2]; ((char*)&a)[2]=t;} +#define swap2(a) { \ + char t=((char*)&a)[0]; ((char*)&a)[0]=((char*)&a)[1]; ((char*)&a)[1]=t;} + + +namespace TNet +{ + /** ************************************************************************** + ** ************************************************************************** + * @brief Aligns a number to a specified base + * @param n Number of type @c _T to align + * @return Aligned value of type @c _T + */ + template<size_t _align, typename _T> + inline _T + align(const _T n) + { + const _T x(_align - 1); + return (n + x) & ~(x); + } + + + /** + * @brief Returns true if architecture is big endian + */ + bool + IsBigEndian(); + + + /** + * @brief Returns true if two numbers are close enough to each other + * + * @param f1 First operand + * @param f2 Second operand + * @param nRounds Expected number of operations prior to this comparison + */ + bool + CloseEnough(const float f1, const float f2, const float nRounds); + + + /** + * @brief Returns true if two numbers are close enough to each other + * + * @param f1 First operand + * @param f2 Second operand + * @param nRounds Expected number of operations prior to this comparison + */ + bool + CloseEnough(const double f1, const double f2, const double nRounds); + + + /** + * @brief Parses a HTK-style string into a C++ std::string readable + * + * @param rIn HTK input string + * @param rOut output parsed string + */ + void + ParseHTKString(const std::string & rIn, std::string & rOut); + + + /** + * @brief Synthesize new file name based on name, path, and extension + * + * @param pOutFileName full ouptut file name + * @param pInFileName file name + * @param pOutDir directory + * @param pOutExt extension + */ + void + MakeHtkFileName(char *pOutFileName, const char* pInFileName, const char *pOutDir, + const char *pOutExt); + + + /** + * @brief Removes the leading and trailing white chars + * + * @param rStr Refference to the string to be processed + * @return Refference to the original string + * + * The white characters are determined by the @c WHITE_CHARS macro defined + * above. + */ + std::string& + Trim(std::string& rStr); + + + char* + StrToUpper(char* pStr); + + char* + ExpandHtkFilterCmd(const char *command, const char *filename, const char* pFilter); + + + template <class T> + std::string to_string(const T& val) + { + std::stringstream ss; + ss << val; + return ss.str(); + } + + inline void + ExpectKeyword(std::istream &i_stream, const char *kwd) + { + std::string token; + i_stream >> token; + if (token != kwd) { + throw std::runtime_error(std::string(kwd) + " expected"); + } + } + + extern const int MATRIX_IOS_FORMAT_IWORD; + + enum MatrixVectorIostreamControlBits { + ACCUMULATE_INPUT = 1, +// BINARY_OUTPUT = 2 + }; + + class MatrixVectorIostreamControl + { + public: + MatrixVectorIostreamControl(enum MatrixVectorIostreamControlBits bitsToBeSet, bool valueToBeSet) + : mBitsToBeSet(bitsToBeSet), mValueToBeSet(valueToBeSet) {} + + static long Flags(std::ios_base &rIos, enum MatrixVectorIostreamControlBits bits) + { return rIos.iword(MATRIX_IOS_FORMAT_IWORD); } + + long mBitsToBeSet; + bool mValueToBeSet; + + friend std::ostream & operator <<(std::ostream &rOs, const MatrixVectorIostreamControl modifier) + { + if(modifier.mValueToBeSet) { + rOs.iword(MATRIX_IOS_FORMAT_IWORD) |= modifier.mBitsToBeSet; + } else { + rOs.iword(MATRIX_IOS_FORMAT_IWORD) &= ~modifier.mBitsToBeSet; + } + return rOs; + } + + friend std::istream & operator >>(std::istream &rIs, const MatrixVectorIostreamControl modifier) + { + if(modifier.mValueToBeSet) { + rIs.iword(MATRIX_IOS_FORMAT_IWORD) |= modifier.mBitsToBeSet; + } else { + rIs.iword(MATRIX_IOS_FORMAT_IWORD) &= ~modifier.mBitsToBeSet; + } + return rIs; + } + }; + + + + +} // namespace TNet + +#ifdef __ICC +#pragma warning (disable: 383) // ICPC remark we don't want. +#pragma warning (disable: 810) // ICPC remark we don't want. +#pragma warning (disable: 981) // ICPC remark we don't want. +#pragma warning (disable: 1418) // ICPC remark we don't want. +#pragma warning (disable: 444) // ICPC remark we don't want. +#pragma warning (disable: 869) // ICPC remark we don't want. +#pragma warning (disable: 1287) // ICPC remark we don't want. +#pragma warning (disable: 279) // ICPC remark we don't want. +#pragma warning (disable: 981) // ICPC remark we don't want. +#endif + +//#ifdef CYGWIN +#if 1 +#undef assert +#ifndef NDEBUG +#define assert(e) ((e) ? (void)0 : assertf(__FILE__, __LINE__, #e)) +#else +#define assert(e) ((void)0) +#endif +void assertf(const char *c, int i, const char *msg); // Just make it possible to break into assert on gdb-- has some kind of bug on cygwin. +#else +#include <cassert> +#endif + +#define assert_throw(e) ((e) ? (void)0 : assertf_throw(__FILE__, __LINE__, #e)) +void assertf_throw(const char *c, int i, const char *msg); + +#define DAN_STYLE_IO + +#endif // ifndef TNet_Common_h + diff --git a/tnet_io/KaldiLib/Error.h b/tnet_io/KaldiLib/Error.h new file mode 100644 index 0000000..2228dde --- /dev/null +++ b/tnet_io/KaldiLib/Error.h @@ -0,0 +1,172 @@ +// +// C++ Interface: %{MODULE} +// +// Description: +// +// +// Author: %{AUTHOR} <%{EMAIL}>, (C) %{YEAR} +// +// Copyright: See COPYING file that comes with this distribution +// +// + +/** @file Error.h + * This header defines several types and functions relating to the + * handling of exceptions in STK. + */ + +#ifndef TNET_Error_h +#define TNET_Error_h + +#include <iostream> +#include <stdexcept> +#include <string> +#include <sstream> + +#include <cstdlib> +#include <execinfo.h> +#include <cstdarg> +#include <cstdio> + +// THESE MACROS TERRIBLY CLASH WITH STK!!!! +// WE MUST USE SAME MACROS! +// +//#define Error(msg) _Error_(__func__, __FILE__, __LINE__, msg) +//#define Warning(msg) _Warning_(__func__, __FILE__, __LINE__, msg) +//#define TraceLog(msg) _TraceLog_(__func__, __FILE__, __LINE__, msg) +// + +#ifndef Error + #define Error(...) _Error_(__func__, __FILE__, __LINE__, __VA_ARGS__) +#endif +#ifndef PError + #define PError(...) _PError_(__func__, __FILE__, __LINE__, __VA_ARGS__) +#endif +#ifndef Warning + #define Warning(...) _Warning_(__func__, __FILE__, __LINE__, __VA_ARGS__) +#endif +#ifndef TraceLog + #define TraceLog(...) _TraceLog_(__func__, __FILE__, __LINE__, __VA_ARGS__) +#endif + +namespace TNet { + + + + /** MyException + * Custom exception class, gets the stacktrace + */ + class MyException + : public std::runtime_error + { + public: + explicit MyException(const std::string& what_arg) throw(); + virtual ~MyException() throw(); + + const char* what() const throw() + { return mWhat.c_str(); } + + private: + std::string mWhat; + }; + + /** + * MyException:: implemenatation + */ + inline + MyException:: + MyException(const std::string& what_arg) throw() + : std::runtime_error(what_arg) + { + mWhat = what_arg; + mWhat += "\nTHE STACKTRACE INSIDE MyException OBJECT IS:\n"; + + void *array[10]; + size_t size; + char **strings; + size_t i; + + size = backtrace (array, 10); + strings = backtrace_symbols (array, size); + + //<< 0th string is the MyException ctor, so ignore and start by 1 + for (i = 1; i < size; i++) { + mWhat += strings[i]; + mWhat += "\n"; + } + + free (strings); + } + + + inline + MyException:: + ~MyException() throw() + { } + + + /** + * @brief Error throwing function (with backtrace) + */ + inline void + _Error_(const char *func, const char *file, int line, const std::string &msg) + { + std::stringstream ss; + ss << "ERROR (" << func << ':' << file << ':' << line << ") " << msg; + throw MyException(ss.str()); + } + + /** + * @brief Throw a formatted error + */ + inline void _PError_(const char *func, const char *file, int line, const char *fmt, ...) { + va_list ap; + char msg[256]; + va_start(ap, fmt); + vsnprintf(msg, sizeof msg, fmt, ap); + va_end(ap); + _Error_(func, file, line, msg); + } + + /** + * @brief Warning handling function + */ + inline void + _Warning_(const char *func, const char *file, int line, const std::string &msg) + { + std::cout << "WARNING (" << func << ':' << file << ':' << line << ") " << msg << std::endl; + } + + inline void + _TraceLog_(const char *func, const char *file, int line, const std::string &msg) + { + std::cout << "INFO (" << func << ':' << file << ':' << line << ") " << msg << std::endl; + std::cout.flush(); + } + + /** + * New kaldi error handling: + * + * class KaldiErrorMessage is invoked from the KALDI_ERROR macro. + * The destructor throws an exception. + */ + class KaldiErrorMessage { + public: + KaldiErrorMessage(const char *func, const char *file, int line) { + this->stream() << "ERROR (" + << func << "():" + << file << ':' << line << ") "; + } + inline std::ostream &stream() { return ss; } + ~KaldiErrorMessage() { throw MyException(ss.str()); } + private: + std::ostringstream ss; + }; + #define KALDI_ERR TNet::KaldiErrorMessage(__func__, __FILE__, __LINE__).stream() + + + +} // namespace TNet + +//#define TNET_Error_h +#endif diff --git a/tnet_io/KaldiLib/Features.cc b/tnet_io/KaldiLib/Features.cc new file mode 100644 index 0000000..64b63e8 --- /dev/null +++ b/tnet_io/KaldiLib/Features.cc @@ -0,0 +1,1798 @@ + +//enable feature repository profiling +#define PROFILING 1 + +#include <sstream> +#include <map> +#include <list> +#include <cstdio> + +#include "Features.h" +#include "Tokenizer.h" +#include "StkMatch.h" +#include "Types.h" + + + +namespace TNet +{ + const char + FeatureRepository:: + mpParmKindNames[13][16] = + { + {"WAVEFORM"}, + {"LPC"}, + {"LPREFC"}, + {"LPCEPSTRA"}, + {"LPDELCEP"}, + {"IREFC"}, + {"MFCC"}, + {"FBANK"}, + {"MELSPEC"}, + {"USER"}, + {"DISCRETE"}, + {"PLP"}, + {"ANON"} + }; + + //*************************************************************************** + //*************************************************************************** + + FileListElem:: + FileListElem(const std::string & rFileName) + { + std::string::size_type pos; + + mLogical = rFileName; + mWeight = 1.0; + + // some slash-backslash replacement hack + for (size_t i = 0; i < mLogical.size(); i++) { + if (mLogical[i] == '\\') { + mLogical[i] = '/'; + } + } + + // read sentence weight definition if any ( physical_file.fea[s,e]{weight} ) + if ((pos = mLogical.find('{')) != std::string::npos) + { + std::string tmp_weight(mLogical.begin() + pos + 1, mLogical.end()); + std::stringstream tmp_ss(tmp_weight); + + tmp_ss >> mWeight; + mLogical.erase(pos); + } + + // look for "=" symbol and if found, split it + if ((pos = mLogical.find('=')) != std::string::npos) + { + // copy all from mLogical[pos+1] till the end to mPhysical + mPhysical.assign(mLogical.begin() + pos + 1, mLogical.end()); + // erase all from pos + 1 till the end from mLogical + mLogical.erase(pos); + // trim the leading and trailing spaces + Trim(mPhysical); + Trim(mLogical); + } + else + { + // trim the leading and trailing spaces + Trim(mLogical); + + mPhysical = mLogical; + } + } + + + //########################################################################### + //########################################################################### + // FeatureRepository section + //########################################################################### + //########################################################################### + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + ReadCepsNormFile( + const char * pFileName, + char ** pLastFileName, + BaseFloat ** vec_buff, + int sampleKind, + CNFileType type, + int coefs) + { + FILE* fp; + int i; + char s1[64]; + char s2[64]; + const char* typeStr = (type == CNF_Mean ? "MEAN" : + type == CNF_Variance ? "VARIANCE" : "VARSCALE"); + + const char* typeStr2 = (type == CNF_Mean ? "CMN" : + type == CNF_Variance ? "CVN" : "VarScale"); + + if (*pLastFileName != NULL && !strcmp(*pLastFileName, pFileName)) { + return; + } + free(*pLastFileName); + *pLastFileName=strdup(pFileName); + *vec_buff = (BaseFloat*) realloc(*vec_buff, coefs * sizeof(BaseFloat)); + + if (*pLastFileName == NULL || *vec_buff== NULL) + throw std::runtime_error("Insufficient memory"); + + if ((fp = fopen(pFileName, "r")) == NULL) { + throw std::runtime_error(std::string("Cannot open ") + typeStr2 + + " pFileName: '" + pFileName + "'"); + } + + if ((type != CNF_VarScale + && (fscanf(fp, " <%64[^>]> <%64[^>]>", s1, s2) != 2 + || strcmp(StrToUpper(s1), "CEPSNORM") + || ReadParmKind(s2, false) != sampleKind)) + || fscanf(fp, " <%64[^>]> %d", s1, &i) != 2 + || strcmp(StrToUpper(s1), typeStr) + || i != coefs) + { + ParmKind2Str(sampleKind, s2); + + //std::cout << "[[[TADY!!!!]]]" << pFileName << "\n" << std::flush; + + throw std::runtime_error(std::string("") + + (type == CNF_VarScale ? "" : "<CEPSNORM> <") + + (type == CNF_VarScale ? "" : s2) + + (type == CNF_VarScale ? "" : ">") + + " <" + typeStr + " ... expected in " + typeStr2 + + " file " + pFileName); + } + + for (i = 0; i < coefs; i++) { + if (fscanf(fp, " "FLOAT_FMT, *vec_buff+i) != 1) { + if (fscanf(fp, "%64s", s2) == 1) { + throw std::runtime_error(std::string("Decimal number expected but '") + + s2 + "' found in " + typeStr2 + " file " + pFileName); + } + else if (feof(fp)) { + throw std::runtime_error(std::string("Unexpected end of ") + + typeStr2 + " file "+ pFileName); + } + else { + throw std::runtime_error(std::string("Cannot read ") + typeStr2 + + " file " + pFileName); + } + } + + if (type == CNF_Variance) + (*vec_buff)[i] = BaseFloat(1 / sqrt((*vec_buff)[i])); + else if (type == CNF_VarScale) + (*vec_buff)[i] = BaseFloat(sqrt((*vec_buff)[i])); + } + + if (fscanf(fp, "%64s", s2) == 1) + { + throw std::runtime_error(std::string("End of file expected but '") + + s2 + "' found in " + typeStr2 + " file " + pFileName); + } + + fclose(fp); + } // ReadCepsNormFile(...) + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + HtkFilter(const char* pFilter, const char* pValue, FeatureRepository& rOut) + { + std::list<FileListElem>::iterator it; + std::string str; + + rOut.mSwapFeatures = mSwapFeatures; + rOut.mStartFrameExt = mStartFrameExt; + rOut.mEndFrameExt = mEndFrameExt; + rOut.mTargetKind = mTargetKind; + rOut.mDerivOrder = mDerivOrder; + rOut.mDerivWinLengths = mDerivWinLengths; + + rOut.mpCvgFile = mpCvgFile; + rOut.mpCmnPath = mpCmnPath; + rOut.mpCmnMask = mpCmnMask; + rOut.mpCvnPath = mpCvnPath; + rOut.mpCvnMask = mpCvnMask; + + rOut.mInputQueue.clear(); + + // go through all records and check the mask + for (it=mInputQueue.begin(); it!= mInputQueue.end(); ++it) { + if (pFilter == NULL + || (ProcessMask(it->Logical(), pFilter, str) && (str == pValue))) { + rOut.mInputQueue.push_back(*it); + } + } + + // set the queue position to the begining + rOut.mInputQueueIterator = mInputQueue.end(); + + rOut.mCurrentIndexFileName = ""; + rOut.mCurrentIndexFileDir = ""; + rOut.mCurrentIndexFileExt = ""; + + mStream.close(); + mStream.clear(); + + rOut.mpLastFileName = NULL; + rOut.mLastFileName = ""; + rOut.mpLastCmnFile = NULL; + rOut.mpLastCvnFile = NULL; + rOut.mpLastCvgFile = NULL; + rOut.mpCmn = NULL; + rOut.mpCvn = NULL; + rOut.mpCvg = NULL; + rOut.mpA = NULL; + rOut.mpB = NULL; + + } + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + HtkSelection(const char* pFilter, std::list< std::string >& rOut) + { + std::map< std::string, bool> aux_map; + std::map< std::string, bool>::iterator map_it; + std::list<FileListElem>::iterator it; + std::string str; + + rOut.clear(); + + if(pFilter != NULL) { + // go through all records and check the mask + for (it=mInputQueue.begin(); it!= mInputQueue.end(); ++it) { + if (ProcessMask(it->Logical(), pFilter, str)) { + aux_map[str] = true; + } + } + } else { + aux_map[std::string("default speaker")] = true; + } + + for (map_it = aux_map.begin(); map_it != aux_map.end(); ++map_it) { + rOut.push_back(map_it->first); + } + } + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + ParmKind2Str(unsigned parmKind, char *pOutString) + { + // :KLUDGE: Absolutely no idea what this is... + if ((parmKind & 0x003F) >= sizeof(mpParmKindNames)/sizeof(mpParmKindNames[0])) + return 0; + + strcpy(pOutString, mpParmKindNames[parmKind & 0x003F]); + + if (parmKind & PARAMKIND_E) strcat(pOutString, "_E"); + if (parmKind & PARAMKIND_N) strcat(pOutString, "_N"); + if (parmKind & PARAMKIND_D) strcat(pOutString, "_D"); + if (parmKind & PARAMKIND_A) strcat(pOutString, "_A"); + if (parmKind & PARAMKIND_C) strcat(pOutString, "_C"); + if (parmKind & PARAMKIND_Z) strcat(pOutString, "_Z"); + if (parmKind & PARAMKIND_K) strcat(pOutString, "_K"); + if (parmKind & PARAMKIND_0) strcat(pOutString, "_0"); + if (parmKind & PARAMKIND_V) strcat(pOutString, "_V"); + if (parmKind & PARAMKIND_T) strcat(pOutString, "_T"); + + return 1; + } + + + // //*************************************************************************** + // //*************************************************************************** + // void + // AddFileListToFeatureRepositories( + // const char* pFileName, + // const char* pFilter, + // std::queue<FeatureRepository *> &featureRepositoryList) + // { + // IStkStream l_stream; + // std::string file_name; + // Tokenizer file_list(pFileName, ","); + // Tokenizer::iterator p_file_name; + + // //:TODO: error if empty featureRepositoryList + // + // for (p_file_name = file_list.begin(); p_file_name != file_list.end(); ++p_file_name) + // { + // // get rid of initial and trailing blanks + // Trim(*p_file_name); + + // // open file name + // l_stream.open(p_file_name->c_str(), std::ios::in, pFilter); + // + // if (!l_stream.good()) { + // //:TODO: + // // Warning or error ... Why warning? -Lukas + // throw std::runtime_error(std::string("Cannot not open list file ") + + // *p_file_name); + // } + + // // read all lines and parse them + // for(;;) + // { + // l_stream >> file_name; + // //:TODO: if(l_stream.badl()) Error() + // // Reading after last token set the fail bit + // if(l_stream.fail()) + // break; + // // we can push_back a std::string as new FileListElem object + // // is created using FileListElem(const std::string&) constructor + // // and logical and physical names are correctly extracted + // featureRepositoryList.front()->mInputQueue.push_back(file_name); + // + // //cycle in the featureRepositoryList + // featureRepositoryList.push(featureRepositoryList.front()); + // featureRepositoryList.pop(); + // } + // l_stream.close(); + // } + // } // AddFileList(const std::string & rFileName) + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + Init( + bool swap, + int extLeft, + int extRight, + int targetKind, + int derivOrder, + int* pDerivWinLen, + const char* pCmnPath, + const char* pCmnMask, + const char* pCvnPath, + const char* pCvnMask, + const char* pCvgFile) + { + mSwapFeatures = swap; + mStartFrameExt = extLeft; + mEndFrameExt = extRight; + mTargetKind = targetKind; + mDerivOrder = derivOrder; + mDerivWinLengths = pDerivWinLen; + mpCmnPath = pCmnPath; + mpCmnMask = pCmnMask; + mpCvnPath = pCvnPath; + mpCvnMask = pCvnMask; + mpCvgFile = pCvgFile; + } // Init() + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + AddFile(const std::string & rFileName) + { + mInputQueue.push_back(rFileName); + } // AddFile(const std::string & rFileName) + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + AddFileList(const char* pFileName, const char* pFilter) + { + IStkStream l_stream; + std::string file_name; + Tokenizer file_list(pFileName, ","); + Tokenizer::iterator p_file_name; + + for (p_file_name = file_list.begin(); p_file_name != file_list.end(); ++p_file_name) + { + // get rid of spaces + Trim(*p_file_name); + + // open the file + l_stream.open(p_file_name->c_str(), std::ios::in, pFilter); + + if (!l_stream.good()) + { + //:TODO: + // Warning or error ... Why warning? -Lukas + throw std::runtime_error(std::string("Cannot not open list file ") + + *p_file_name); + } + // read all lines and parse them + for(;;) + { + l_stream >> file_name; + //:TODO: if(l_stream.badl()) Error() + // Reading after last token set the fail bit + if(l_stream.fail()) + break; + // we can push_back a std::string as new FileListElem object + // is created using FileListElem(const std::string&) constructor + // and logical and physical names are correctly extracted + mInputQueue.push_back(file_name); + } + l_stream.close(); + } + } // AddFileList(const std::string & rFileName) + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + MoveNext() + { + assert (mInputQueueIterator != mInputQueue.end()); + mInputQueueIterator++; + } // ReadFullMatrix(Matrix<BaseFloat>& rMatrix) + + + //*************************************************************************** + //*************************************************************************** + bool + FeatureRepository:: + ReadFullMatrix(Matrix<BaseFloat>& rMatrix) + { + // clear the matrix + rMatrix.Destroy(); + + // extract index file name + if (!mCurrentIndexFileDir.empty()) + { + char tmp_name[mCurrentIndexFileDir.length() + + mCurrentIndexFileExt.length() + + mInputQueueIterator->Physical().length()]; + + MakeHtkFileName(tmp_name, mInputQueueIterator->Physical().c_str(), + mCurrentIndexFileDir.c_str(), mCurrentIndexFileExt.c_str()); + + mCurrentIndexFileName = tmp_name; + } + else + mCurrentIndexFileName = ""; + + //get the 3-letter suffix + int pos_last_three_chars = mInputQueueIterator->Physical().size() - 3; + if (pos_last_three_chars < 0) pos_last_three_chars = 0; + //read the gzipped ascii features + if (mInputQueueIterator->Physical().substr(pos_last_three_chars) == ".gz") { + return ReadGzipAsciiFeatures(*mInputQueueIterator, rMatrix); + } + + // read the matrix and return the result + return ReadHTKFeatures(*mInputQueueIterator, rMatrix); + } // ReadFullMatrix(Matrix<BaseFloat>& rMatrix) + + + + //*************************************************************************** + //*************************************************************************** + bool + FeatureRepository:: + WriteFeatureMatrix(const Matrix<BaseFloat>& rMatrix, const std::string& filename, int targetKind, int samplePeriod) + { + FILE* fp = fopen(filename.c_str(),"w"); + if(NULL == fp) { Error(std::string("Cannot create file:") + filename); return false; } + + WriteHTKFeatures(fp, samplePeriod, targetKind, mSwapFeatures, const_cast<Matrix<BaseFloat>&>(rMatrix)); + + fclose(fp); + + return true; + } + + + //*************************************************************************** + //*************************************************************************** + // private: + int + FeatureRepository:: + ReadHTKHeader() + { + // TODO + // Change this... We should read from StkStream + FILE* fp = mStream.fp(); + + if (!fread(&mHeader.mNSamples, sizeof(INT_32), 1, fp)) return -1; + if (!fread(&mHeader.mSamplePeriod, sizeof(INT_32), 1, fp)) return -1; + if (!fread(&mHeader.mSampleSize, sizeof(INT_16), 1, fp)) return -1; + if (!fread(&mHeader.mSampleKind, sizeof(UINT_16), 1, fp)) return -1; + + if (mSwapFeatures) + { + swap4(mHeader.mNSamples); + swap4(mHeader.mSamplePeriod); + swap2(mHeader.mSampleSize); + swap2(mHeader.mSampleKind); + } + + if (mHeader.mSamplePeriod < 0 + || mHeader.mSamplePeriod > 1000000 + || mHeader.mNSamples < 0 + || mHeader.mSampleSize < 0) + { + return -1; + } + + return 0; + } + + + //*************************************************************************** + //*************************************************************************** + // private: + int + FeatureRepository:: + ReadHTKFeature( + BaseFloat* pIn, + size_t feaLen, + bool decompress, + BaseFloat* pScale, + BaseFloat* pBias) + { + FILE* fp = mStream.fp(); + + size_t i; + + if (decompress) + { + INT_16 s; + // BaseFloat pScale = (xmax - xmin) / (2*32767); + // BaseFloat pBias = (xmax + xmin) / 2; + + for (i = 0; i < feaLen; i++) + { + if (fread(&s, sizeof(INT_16), 1, fp) != 1) + return -1; + + if (mSwapFeatures) swap2(s); + pIn[i] = ((BaseFloat)s + pBias[i]) / pScale[i]; + } + + return 0; + } + +#if !DOUBLEPRECISION + if (fread(pIn, sizeof(FLOAT_32), feaLen, fp) != feaLen) + return -1; + + if (mSwapFeatures) + for (i = 0; i < feaLen; i++) + swap4(pIn[i]); +#else + float f; + + for (i = 0; i < feaLen; i++) + { + if (fread(&f, sizeof(FLOAT_32), 1, fp) != 1) + return -1; + + if (mSwapFeatures) + swap4(f); + + pIn[i] = f; + } +#endif + return 0; + } // int ReadHTKFeature + + + + //*************************************************************************** + //*************************************************************************** +/* bool + FeatureRepository:: + ReadHTKFeatures(const std::string& rFileName, Matrix<BaseFloat>& rFeatureMatrix) + { + std::string file_name(rFileName); + std::string cmn_file_name; + std::string cvn_file_name; + + int ext_left = mStartFrameExt; + int ext_right = mEndFrameExt; + int from_frame; + int to_frame; + int tot_frames; + int trg_vec_size; + int src_vec_size; + int src_deriv_order; + int lo_src_tgz_deriv_order; + int i; + int j; + int k; + int e; + int coefs; + int trg_E; + int trg_0; + int trg_N; + int src_E; + int src_0; + int src_N; + int comp; + int coef_size; + char* chptr; + + + + // read frame range definition if any ( physical_file.fea[s,e] ) + if ((chptr = strrchr(file_name.c_str(), '[')) == NULL || + ((i=0), sscanf(chptr, "[%d,%d]%n", &from_frame, &to_frame, &i), + chptr[i] != '\0')) + { + chptr = NULL; + } + + if (chptr != NULL) + *chptr = '\0'; + + // Experimental changes... + // if ((strcmp(file_name.c_str(), "-")) + // && (mpLastFileName != NULL) + // && (!strcmp(mpLastFileName, file_name.c_str()))) + // { + // mHeader = mLastHeader; + // } + // else + // { + // if (mpLastFileName) + // { + // //if (mpFp != stdin) + // // fclose(mpFp); + // mStream.close(); + // + // free(mpLastFileName); + // mpLastFileName = NULL; + // } + + if ((file_name != "-" ) + && (!mLastFileName.empty()) + && (mLastFileName == file_name)) + { + mHeader = mLastHeader; + } + else + { + if (!mLastFileName.empty()) + { + mStream.close(); + mLastFileName = ""; + } + + + // open the feature file + mStream.open(file_name.c_str(), ios::binary); + if (!mStream.good()) + { + Error("Cannot open feature file: '%s'", file_name.c_str()); + } + + + if (ReadHTKHeader()) + Error("Invalid HTK header in feature file: '%s'", file_name.c_str()); + + if (mHeader.mSampleKind & PARAMKIND_C) + { + // File is in compressed form, scale and pBias vectors + // are appended after HTK header. + + int coefs = mHeader.mSampleSize/sizeof(INT_16); + mpA = (BaseFloat*) realloc(mpA, coefs * sizeof(BaseFloat)); + mpB = (BaseFloat*) realloc(mpB, coefs * sizeof(BaseFloat)); + if (mpA == NULL || mpB == NULL) Error("Insufficient memory"); + + e = ReadHTKFeature(mpA, coefs, 0, 0, 0); + e |= ReadHTKFeature(mpB, coefs, 0, 0, 0); + + if (e) + Error("Cannot read feature file: '%s'", file_name.c_str()); + + mHeader.mNSamples -= 2 * sizeof(FLOAT_32) / sizeof(INT_16); + } + + // remember current settings + mLastFileName = file_name; + mLastHeader = mHeader; + } + + if (chptr != NULL) + *chptr = '['; + + if (chptr == NULL) + { // Range [s,e] was not specified + from_frame = 0; + to_frame = mHeader.mNSamples-1; + } + + src_deriv_order = PARAMKIND_T & mHeader.mSampleKind ? 3 : + PARAMKIND_A & mHeader.mSampleKind ? 2 : + PARAMKIND_D & mHeader.mSampleKind ? 1 : 0; + src_E = (PARAMKIND_E & mHeader.mSampleKind) != 0; + src_0 = (PARAMKIND_0 & mHeader.mSampleKind) != 0; + src_N = ((PARAMKIND_N & mHeader.mSampleKind) != 0) * (src_E + src_0); + comp = PARAMKIND_C & mHeader.mSampleKind; + + mHeader.mSampleKind &= ~PARAMKIND_C; + + if (mTargetKind == PARAMKIND_ANON) + { + mTargetKind = mHeader.mSampleKind; + } + else if ((mTargetKind & 077) == PARAMKIND_ANON) + { + mTargetKind &= ~077; + mTargetKind |= mHeader.mSampleKind & 077; + } + + trg_E = (PARAMKIND_E & mTargetKind) != 0; + trg_0 = (PARAMKIND_0 & mTargetKind) != 0; + trg_N =((PARAMKIND_N & mTargetKind) != 0) * (trg_E + trg_0); + + coef_size = comp ? sizeof(INT_16) : sizeof(FLOAT_32); + coefs = (mHeader.mSampleSize/coef_size + src_N) / + (src_deriv_order+1) - src_E - src_0; + src_vec_size = (coefs + src_E + src_0) * (src_deriv_order+1) - src_N; + + //Is coefs dividable by 1 + number of derivatives specified in header + if (src_vec_size * coef_size != mHeader.mSampleSize) + { + Error("Invalid HTK header in feature file: '%s'. " + "mSampleSize do not match with parmKind", file_name.c_str()); + } + + if (mDerivOrder < 0) + mDerivOrder = src_deriv_order; + + + if ((!src_E && trg_E) || (!src_0 && trg_0) || (src_N && !trg_N) || + (trg_N && !trg_E && !trg_0) || (trg_N && !mDerivOrder) || + (src_N && !src_deriv_order && mDerivOrder) || + ((mHeader.mSampleKind & 077) != (mTargetKind & 077) && + (mHeader.mSampleKind & 077) != PARAMKIND_ANON)) + { + char srcParmKind[64]; + char trgParmKind[64]; + + ParmKind2Str(mHeader.mSampleKind, srcParmKind); + ParmKind2Str(mTargetKind, trgParmKind); + Error("Cannot convert %s to %s", srcParmKind, trgParmKind); + } + + lo_src_tgz_deriv_order = LOWER_OF(src_deriv_order, mDerivOrder); + trg_vec_size = (coefs + trg_E + trg_0) * (mDerivOrder+1) - trg_N; + + i = LOWER_OF(from_frame, mStartFrameExt); + from_frame -= i; + ext_left -= i; + + i = LOWER_OF(mHeader.mNSamples-to_frame-1, mEndFrameExt); + to_frame += i; + ext_right -= i; + + if (from_frame > to_frame || from_frame >= mHeader.mNSamples || to_frame< 0) + Error("Invalid frame range for feature file: '%s'", file_name.c_str()); + + tot_frames = to_frame - from_frame + 1 + ext_left + ext_right; + + // initialize matrix + rFeatureMatrix.Init(tot_frames, trg_vec_size); + + // fill the matrix with features + for (i = 0; i <= to_frame - from_frame; i++) + { + BaseFloat* A = mpA; + BaseFloat* B = mpB; + BaseFloat* mxPtr = rFeatureMatrix[i+ext_left]; + + // seek to the desired position + fseek(mStream.fp(), + sizeof(HtkHeader) + (comp ? src_vec_size * 2 * sizeof(FLOAT_32) : 0) + + (from_frame + i) * src_vec_size * coef_size, + SEEK_SET); + + e = ReadHTKFeature(mxPtr, coefs, comp, A, B); + + mxPtr += coefs; + A += coefs; + B += coefs; + + if (src_0 && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_0 && !trg_N) mxPtr++; + if (src_E && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_E && !trg_N) mxPtr++; + + for (j = 0; j < lo_src_tgz_deriv_order; j++) + { + e |= ReadHTKFeature(mxPtr, coefs, comp, A, B); + mxPtr += coefs; + A += coefs; + B += coefs; + + if (src_0) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_0) mxPtr++; + if (src_E) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_E) mxPtr++; + } + + if (e) + Error("Cannot read feature file: '%s' frame %d/%d", file_name.c_str(), + i, to_frame - from_frame + 1); + } + + // From now, coefs includes also trg_0 + trg_E ! + coefs += trg_0 + trg_E; + + // If extension of the matrix to the left or to the right is required, + // perform it here + for (i = 0; i < ext_left; i++) + { + memcpy(rFeatureMatrix[i], + rFeatureMatrix[ext_left], + (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat)); + } + + for (i = tot_frames - ext_right; i < tot_frames; i++) + { + memcpy(rFeatureMatrix[i], + rFeatureMatrix[tot_frames - ext_right - 1], + (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat)); + } + + // Sentence cepstral mean normalization + if( (mpCmnPath == NULL) + && !(PARAMKIND_Z & mHeader.mSampleKind) + && (PARAMKIND_Z & mTargetKind)) + { + // for each coefficient + for(j=0; j < coefs; j++) + { + BaseFloat norm = 0.0; + for(i=0; i < tot_frames; i++) // for each frame + { + norm += rFeatureMatrix[i][j - trg_N]; + //norm += fea_mx[i*trg_vec_size - trg_N + j]; + } + + norm /= tot_frames; + + for(i=0; i < tot_frames; i++) // for each frame + rFeatureMatrix[i][j - trg_N] -= norm; + //fea_mx[i*trg_vec_size - trg_N + j] -= norm; + } + } + + // Compute missing derivatives + for (; src_deriv_order < mDerivOrder; src_deriv_order++) + { + int winLen = mDerivWinLengths[src_deriv_order]; + BaseFloat norm = 0.0; + + for (k = 1; k <= winLen; k++) + { + norm += 2 * k * k; + } + + // for each frame + for (i=0; i < tot_frames; i++) + { + // for each coefficient + for (j=0; j < coefs; j++) + { + //BaseFloat* src = fea_mx + i*trg_vec_size + src_deriv_order*coefs - trg_N + j; + BaseFloat* src = &rFeatureMatrix[i][src_deriv_order*coefs - trg_N + j]; + + *(src + coefs) = 0.0; + + if (i < winLen || i >= tot_frames-winLen) + { // boundaries need special treatment + for (k = 1; k <= winLen; k++) + { + *(src+coefs) += k*(src[ LOWER_OF(tot_frames-1-i,k)*rFeatureMatrix.Stride()] + -src[-LOWER_OF(i, k)*rFeatureMatrix.Stride()]); + } + } + else + { // otherwise use more efficient code + for (k = 1; k <= winLen; k++) + { + *(src+coefs) += k*(src[ k * rFeatureMatrix.Stride()] + -src[-k * rFeatureMatrix.Stride()]); + } + } + *(src + coefs) /= norm; + } + } + } + + mHeader.mNSamples = tot_frames; + mHeader.mSampleSize = trg_vec_size * sizeof(FLOAT_32); + mHeader.mSampleKind = mTargetKind & ~(PARAMKIND_D | PARAMKIND_A | PARAMKIND_T); + + + //////////////////////////////////////////////////////////////////////////// + /////////////// Cepstral mean and variance normalization /////////////////// + //////////////////////////////////////////////////////////////////////////// + //......................................................................... + if (mpCmnPath != NULL + && mpCmnMask != NULL) + { + // retrieve file name + ProcessMask(file_name, mpCmnMask, cmn_file_name); + // add the path correctly + cmn_file_name.insert(0, "/"); + cmn_file_name.insert(0, mpCmnPath); + + // read the file + ReadCepsNormFile(cmn_file_name.c_str(), &mpLastCmnFile, &mpCmn, + mHeader.mSampleKind & ~PARAMKIND_Z, CNF_Mean, coefs); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < coefs; j++) + { + rFeatureMatrix[i][j - trg_N] -= mpCmn[j]; + } + } + } + + mHeader.mSampleKind |= mDerivOrder==3 ? PARAMKIND_D | PARAMKIND_A | PARAMKIND_T : + mDerivOrder==2 ? PARAMKIND_D | PARAMKIND_A : + mDerivOrder==1 ? PARAMKIND_D : 0; + + //......................................................................... + if (mpCvnPath != NULL + && mpCvnMask != NULL) + { + // retrieve file name + ProcessMask(file_name, mpCvnMask, cvn_file_name); + // add the path correctly + cvn_file_name.insert(0, "/"); + cvn_file_name.insert(0, mpCvnPath); + + // read the file + ReadCepsNormFile(cvn_file_name.c_str(), &mpLastCvnFile, &mpCvn, + mHeader.mSampleKind, CNF_Variance, trg_vec_size); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < trg_vec_size; j++) + { + rFeatureMatrix[i][j - trg_N] *= mpCvn[j]; + } + } + } + + //......................................................................... + // process the global covariance file + if (mpCvgFile != NULL) + { + ReadCepsNormFile(mpCvgFile, &mpLastCvgFile, &mpCvg, + -1, CNF_VarScale, trg_vec_size); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < trg_vec_size; j++) + { + rFeatureMatrix[i][j - trg_N] *= mpCvg[j]; + } + } + } + + return true; + } +*/ + + //*************************************************************************** + //*************************************************************************** + + + + + + //*************************************************************************** + //*************************************************************************** + bool + FeatureRepository:: + ReadHTKFeatures(const FileListElem& rFileNameRecord, + Matrix<BaseFloat>& rFeatureMatrix) + { + std::string file_name(rFileNameRecord.Physical()); + std::string cmn_file_name; + std::string cvn_file_name; + + int ext_left = mStartFrameExt; + int ext_right = mEndFrameExt; + int from_frame; + int to_frame; + int tot_frames; + int trg_vec_size; + int src_vec_size; + int src_deriv_order; + int lo_src_tgz_deriv_order; + int i; + int j; + int k; + int e; + int coefs; + int trg_E; + int trg_0; + int trg_N; + int src_E; + int src_0; + int src_N; + int comp; + int coef_size; + char* chptr; + + + TIMER_START(mTim); + + // read frame range definition if any ( physical_file.fea[s,e] ) + if ((chptr = strrchr((char*)file_name.c_str(), '[')) == NULL || + ((i=0), sscanf(chptr, "[%d,%d]%n", &from_frame, &to_frame, &i), + chptr[i] != '\0')) + { + chptr = NULL; + } + + if (chptr != NULL) + *chptr = '\0'; + + + if ((file_name != "-" ) + && (!mLastFileName.empty()) + && (mLastFileName == file_name)) + { + mHeader = mLastHeader; + } + else + { + if (!mLastFileName.empty()) + { + mStream.close(); + mLastFileName = ""; + } + + + // open the feature file + mStream.open(file_name.c_str(), std::ios::binary); + if (!mStream.good()) + { + throw std::runtime_error(std::string("Cannot open feature file: '") + + file_name.c_str() + "'"); + } + + + if (ReadHTKHeader()) { + throw std::runtime_error(std::string("Invalid HTK header in feature file: '") + + file_name.c_str() + "'"); + } + + if (mHeader.mSampleKind & PARAMKIND_C) + { + // File is in compressed form, scale and pBias vectors + // are appended after HTK header. + coefs = mHeader.mSampleSize/sizeof(INT_16); + + mpA = (BaseFloat*) realloc(mpA, coefs * sizeof(BaseFloat)); + mpB = (BaseFloat*) realloc(mpB, coefs * sizeof(BaseFloat)); + + if (mpA == NULL || mpB == NULL) { + throw std::runtime_error("Insufficient memory"); + } + + e = ReadHTKFeature(mpA, coefs, 0, 0, 0); + e |= ReadHTKFeature(mpB, coefs, 0, 0, 0); + + if (e) { + throw std::runtime_error(std::string("Cannot read feature file: '") + + file_name.c_str() + "'"); + } + + mHeader.mNSamples -= 2 * sizeof(FLOAT_32) / sizeof(INT_16); + } + + // remember current settings + mLastFileName = file_name; + mLastHeader = mHeader; + } + + if (chptr != NULL) { + *chptr = '['; + } + + if (chptr == NULL) { + // Range [s,e] was not specified + from_frame = 0; + to_frame = mHeader.mNSamples-1; + } + + src_deriv_order = PARAMKIND_T & mHeader.mSampleKind ? 3 : + PARAMKIND_A & mHeader.mSampleKind ? 2 : + PARAMKIND_D & mHeader.mSampleKind ? 1 : 0; + src_E = (PARAMKIND_E & mHeader.mSampleKind) != 0; + src_0 = (PARAMKIND_0 & mHeader.mSampleKind) != 0; + src_N = ((PARAMKIND_N & mHeader.mSampleKind) != 0) * (src_E + src_0); + comp = PARAMKIND_C & mHeader.mSampleKind; + + mHeader.mSampleKind &= ~PARAMKIND_C; + + if (mTargetKind == PARAMKIND_ANON) + { + mTargetKind = mHeader.mSampleKind; + } + else if ((mTargetKind & 077) == PARAMKIND_ANON) + { + mTargetKind &= ~077; + mTargetKind |= mHeader.mSampleKind & 077; + } + + trg_E = (PARAMKIND_E & mTargetKind) != 0; + trg_0 = (PARAMKIND_0 & mTargetKind) != 0; + trg_N =((PARAMKIND_N & mTargetKind) != 0) * (trg_E + trg_0); + + coef_size = comp ? sizeof(INT_16) : sizeof(FLOAT_32); + coefs = (mHeader.mSampleSize/coef_size + src_N) / + (src_deriv_order+1) - src_E - src_0; + src_vec_size = (coefs + src_E + src_0) * (src_deriv_order+1) - src_N; + + //Is coefs dividable by 1 + number of derivatives specified in header + if (src_vec_size * coef_size != mHeader.mSampleSize) + { + throw std::runtime_error(std::string("Invalid HTK header in feature file: '") + + file_name + "' mSampleSize do not match with parmKind"); + } + + if (mDerivOrder < 0) + mDerivOrder = src_deriv_order; + + + if ((!src_E && trg_E) || (!src_0 && trg_0) || (src_N && !trg_N) || + (trg_N && !trg_E && !trg_0) || (trg_N && !mDerivOrder) || + (src_N && !src_deriv_order && mDerivOrder) || + ((mHeader.mSampleKind & 077) != (mTargetKind & 077) && + (mHeader.mSampleKind & 077) != PARAMKIND_ANON)) + { + char srcParmKind[64]; + char trgParmKind[64]; + memset(srcParmKind,0,64); + memset(trgParmKind,0,64); + + ParmKind2Str(mHeader.mSampleKind, srcParmKind); + ParmKind2Str(mTargetKind, trgParmKind); + throw std::runtime_error(std::string("Cannot convert ") + srcParmKind + + " to " + trgParmKind); + } + + lo_src_tgz_deriv_order = std::min(src_deriv_order, mDerivOrder); + trg_vec_size = (coefs + trg_E + trg_0) * (mDerivOrder+1) - trg_N; + + i = std::min(from_frame, mStartFrameExt); + from_frame -= i; + ext_left -= i; + + i = std::min(mHeader.mNSamples-to_frame-1, mEndFrameExt); + to_frame += i; + ext_right -= i; + + if (from_frame > to_frame || from_frame >= mHeader.mNSamples || to_frame< 0) + throw std::runtime_error(std::string("Invalid frame range for feature file: '") + + file_name.c_str() + "'"); + + tot_frames = to_frame - from_frame + 1 + ext_left + ext_right; + + + TIMER_END(mTim,mTimeOpen); + + + // initialize matrix + rFeatureMatrix.Init(tot_frames, trg_vec_size, false); + + // fill the matrix with features + for (i = 0; i <= to_frame - from_frame; i++) + { + BaseFloat* A = mpA; + BaseFloat* B = mpB; + BaseFloat* mxPtr = rFeatureMatrix.pRowData(i+ext_left); + + TIMER_START(mTim); + // seek to the desired position + fseek(mStream.fp(), + sizeof(HtkHeader) + (comp ? src_vec_size * 2 * sizeof(FLOAT_32) : 0) + + (from_frame + i) * src_vec_size * coef_size, + SEEK_SET); + TIMER_END(mTim,mTimeSeek); + + TIMER_START(mTim); + // read + e = ReadHTKFeature(mxPtr, coefs, comp, A, B); + TIMER_END(mTim,mTimeRead); + + mxPtr += coefs; + A += coefs; + B += coefs; + + if (src_0 && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_0 && !trg_N) mxPtr++; + if (src_E && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_E && !trg_N) mxPtr++; + + for (j = 0; j < lo_src_tgz_deriv_order; j++) + { + e |= ReadHTKFeature(mxPtr, coefs, comp, A, B); + mxPtr += coefs; + A += coefs; + B += coefs; + + if (src_0) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_0) mxPtr++; + if (src_E) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_E) mxPtr++; + } + + if (e) { + std::cout << mHeader.mNSamples << "\n"; + std::cout << 2 * sizeof(FLOAT_32) / sizeof(INT_16) << "\n"; + std::cout << "from" << from_frame << "to" << to_frame << "i" << i << "\n"; + + std::ostringstream s; + s << i << "/" << to_frame - from_frame + 1, s.str(); + throw std::runtime_error(std::string("Cannot read feature file: '") + + file_name + "' frame " + s.str()); + } + } + + // From now, coefs includes also trg_0 + trg_E ! + coefs += trg_0 + trg_E; + + // If extension of the matrix to the left or to the right is required, + // perform it here + for (i = 0; i < ext_left; i++) + { + memcpy(rFeatureMatrix.pRowData(i), + rFeatureMatrix.pRowData(ext_left), + (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat)); + } + + for (i = tot_frames - ext_right; i < tot_frames; i++) + { + memcpy(rFeatureMatrix.pRowData(i), + rFeatureMatrix.pRowData(tot_frames - ext_right - 1), + (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat)); + } + + // Sentence cepstral mean normalization + if( (mpCmnPath == NULL) + && !(PARAMKIND_Z & mHeader.mSampleKind) + && (PARAMKIND_Z & mTargetKind)) + { + // for each coefficient + for(j=0; j < coefs; j++) + { + BaseFloat norm = 0.0; + for(i=0; i < tot_frames; i++) // for each frame + { + norm += rFeatureMatrix[i][j - trg_N]; + //norm += fea_mx[i*trg_vec_size - trg_N + j]; + } + + norm /= tot_frames; + + for(i=0; i < tot_frames; i++) // for each frame + rFeatureMatrix[i][j - trg_N] -= norm; + //fea_mx[i*trg_vec_size - trg_N + j] -= norm; + } + } + + // Compute missing derivatives + for (; src_deriv_order < mDerivOrder; src_deriv_order++) + { + int winLen = mDerivWinLengths[src_deriv_order]; + BaseFloat norm = 0.0; + + for (k = 1; k <= winLen; k++) + { + norm += 2 * k * k; + } + + // for each frame + for (i=0; i < tot_frames; i++) + { + // for each coefficient + for (j=0; j < coefs; j++) + { + //BaseFloat* src = fea_mx + i*trg_vec_size + src_deriv_order*coefs - trg_N + j; + BaseFloat* src = &rFeatureMatrix[i][src_deriv_order*coefs - trg_N + j]; + + *(src + coefs) = 0.0; + + if (i < winLen || i >= tot_frames-winLen) + { // boundaries need special treatment + for (k = 1; k <= winLen; k++) + { + *(src+coefs) += k*(src[ std::min(tot_frames-1-i,k)*rFeatureMatrix.Stride()] + -src[-std::min(i, k)*rFeatureMatrix.Stride()]); + } + } + else + { // otherwise use more efficient code + for (k = 1; k <= winLen; k++) + { + *(src+coefs) += k*(src[ k * rFeatureMatrix.Stride()] + -src[-k * rFeatureMatrix.Stride()]); + } + } + *(src + coefs) /= norm; + } + } + } + + mHeader.mNSamples = tot_frames; + mHeader.mSampleSize = trg_vec_size * sizeof(FLOAT_32); + mHeader.mSampleKind = mTargetKind & ~(PARAMKIND_D | PARAMKIND_A | PARAMKIND_T); + + + TIMER_START(mTim); + //////////////////////////////////////////////////////////////////////////// + /////////////// Cepstral mean and variance normalization /////////////////// + //////////////////////////////////////////////////////////////////////////// + //......................................................................... + if (mpCmnPath != NULL + && mpCmnMask != NULL) + { + // retrieve file name + ProcessMask(rFileNameRecord.Logical(), mpCmnMask, cmn_file_name); + // add the path correctly + + if(cmn_file_name == "") { + throw std::runtime_error("CMN Matching failed"); + } + + cmn_file_name.insert(0, "/"); + cmn_file_name.insert(0, mpCmnPath); + + // read the file + ReadCepsNormFile(cmn_file_name.c_str(), &mpLastCmnFile, &mpCmn, + mHeader.mSampleKind & ~PARAMKIND_Z, CNF_Mean, coefs); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < coefs; j++) + { + rFeatureMatrix[i][j - trg_N] -= mpCmn[j]; + } + } + } + + mHeader.mSampleKind |= mDerivOrder==3 ? PARAMKIND_D | PARAMKIND_A | PARAMKIND_T : + mDerivOrder==2 ? PARAMKIND_D | PARAMKIND_A : + mDerivOrder==1 ? PARAMKIND_D : 0; + + //......................................................................... + if (mpCvnPath != NULL + && mpCvnMask != NULL) + { + // retrieve file name + ProcessMask(rFileNameRecord.Logical(), mpCvnMask, cvn_file_name); + // add the path correctly + cvn_file_name.insert(0, "/"); + cvn_file_name.insert(0, mpCvnPath); + + // read the file + ReadCepsNormFile(cvn_file_name.c_str(), &mpLastCvnFile, &mpCvn, + mHeader.mSampleKind, CNF_Variance, trg_vec_size); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < trg_vec_size; j++) + { + rFeatureMatrix[i][j - trg_N] *= mpCvn[j]; + } + } + } + + //......................................................................... + // process the global covariance file + if (mpCvgFile != NULL) + { + ReadCepsNormFile(mpCvgFile, &mpLastCvgFile, &mpCvg, + -1, CNF_VarScale, trg_vec_size); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < trg_vec_size; j++) + { + rFeatureMatrix[i][j - trg_N] *= mpCvg[j]; + } + } + } + + TIMER_END(mTim,mTimeNormalize); + + return true; + } + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + ReadParmKind(const char *str, bool checkBrackets) + { + unsigned int i; + int parmKind =0; + int slen = strlen(str); + + if (checkBrackets) + { + if (str[0] != '<' || str[slen-1] != '>') return -1; + str++; slen -= 2; + } + + for (; slen >= 0 && str[slen-2] == '_'; slen -= 2) + { + parmKind |= str[slen-1] == 'E' ? PARAMKIND_E : + str[slen-1] == 'N' ? PARAMKIND_N : + str[slen-1] == 'D' ? PARAMKIND_D : + str[slen-1] == 'A' ? PARAMKIND_A : + str[slen-1] == 'C' ? PARAMKIND_C : + str[slen-1] == 'Z' ? PARAMKIND_Z : + str[slen-1] == 'K' ? PARAMKIND_K : + str[slen-1] == '0' ? PARAMKIND_0 : + str[slen-1] == 'V' ? PARAMKIND_V : + str[slen-1] == 'T' ? PARAMKIND_T : -1; + + if (parmKind == -1) return -1; + } + + for (i = 0; i < sizeof(mpParmKindNames) / sizeof(char*); i++) + { + if (!strncmp(str, mpParmKindNames[i], slen)) + return parmKind | i; + } + return -1; + } + + + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + WriteHTKHeader (FILE * pOutFp, HtkHeader header, bool swap) + { + int cc; + + if (swap) { + swap4(header.mNSamples); + swap4(header.mSamplePeriod); + swap2(header.mSampleSize); + swap2(header.mSampleKind); + } + + fseek (pOutFp, 0L, SEEK_SET); + cc = fwrite(&header, sizeof(HtkHeader), 1, pOutFp); + + if (swap) { + swap4(header.mNSamples); + swap4(header.mSamplePeriod); + swap2(header.mSampleSize); + swap2(header.mSampleKind); + } + + return cc == 1 ? 0 : -1; + } + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + WriteHTKFeature( + FILE * pOutFp, + FLOAT * pOut, + size_t feaLen, + bool swap, + bool compress, + FLOAT* pScale, + FLOAT* pBias) + { + size_t i; + size_t cc = 0; + + + if (compress) + { + INT_16 s; + + for (i = 0; i < feaLen; i++) + { + s = pOut[i] * pScale[i] - pBias[i]; + if (swap) + swap2(s); + cc += fwrite(&s, sizeof(INT_16), 1, pOutFp); + } + + } else { + #if !DOUBLEPRECISION + if (swap) + for (i = 0; i < feaLen; i++) + swap4(pOut[i]); + + cc = fwrite(pOut, sizeof(FLOAT_32), feaLen, pOutFp); + + if (swap) + for (i = 0; i < feaLen; i++) + swap4(pOut[i]); + #else + FLOAT_32 f; + + for (i = 0; i < feaLen; i++) + { + f = pOut[i]; + if (swap) + swap4(f); + cc += fwrite(&f, sizeof(FLOAT_32), 1, pOutFp); + } + #endif + } + return cc == feaLen ? 0 : -1; + } + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + WriteHTKFeatures( + FILE * pOutFp, + FLOAT * pOut, + int nCoeffs, + int nSamples, + int samplePeriod, + int targetKind, + bool swap) + { + HtkHeader header; + int i, j; + FLOAT *pScale = NULL; + FLOAT *pBias = NULL; + + header.mNSamples = nSamples + ((targetKind & PARAMKIND_C) ? 2 * sizeof(FLOAT_32) / sizeof(INT_16) : 0); + header.mSamplePeriod = samplePeriod; + header.mSampleSize = nCoeffs * ((targetKind & PARAMKIND_C) ? sizeof(INT_16) : sizeof(FLOAT_32));; + header.mSampleKind = targetKind; + + WriteHTKHeader (pOutFp, header, swap); + + if(targetKind & PARAMKIND_C) { + pScale = (FLOAT*) malloc(nCoeffs * sizeof(FLOAT)); + pBias = (FLOAT*) malloc(nCoeffs * sizeof(FLOAT)); + if (pScale == NULL || pBias == NULL) Error("Insufficient memory"); + + for(i = 0; i < nCoeffs; i++) { + float xmin, xmax; + xmin = xmax = pOut[i]; + for(j = 1; j < nSamples; j++) { + if(pOut[j*nCoeffs+i] > xmax) xmax = pOut[j*nCoeffs+i]; + if(pOut[j*nCoeffs+i] < xmin) xmin = pOut[j*nCoeffs+i]; + } + pScale[i] = (2*32767) / (xmax - xmin); + pBias[i] = pScale[i] * (xmax + xmin) / 2; + + + } + if (WriteHTKFeature(pOutFp, pScale, nCoeffs, swap, false, 0, 0) + || WriteHTKFeature(pOutFp, pBias, nCoeffs, swap, false, 0, 0)) { + return -1; + } + } + for(j = 0; j < nSamples; j++) { + if (WriteHTKFeature(pOutFp, &pOut[j*nCoeffs], nCoeffs, swap, targetKind & PARAMKIND_C, pScale, pBias)) { + return -1; + } + } + return 0; + } + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + WriteHTKFeatures( + FILE * pOutFp, + int samplePeriod, + int targetKind, + bool swap, + Matrix<BaseFloat>& rFeatureMatrix) + { + HtkHeader header; + size_t i, j; + FLOAT *p_scale = NULL; + FLOAT *p_bias = NULL; + size_t n_samples = rFeatureMatrix.Rows(); + size_t n_coeffs = rFeatureMatrix.Cols(); + + header.mNSamples = n_samples + ((targetKind & PARAMKIND_C) ? 2 * sizeof(FLOAT_32) / sizeof(INT_16) : 0); + header.mSamplePeriod = samplePeriod; + header.mSampleSize = n_coeffs * ((targetKind & PARAMKIND_C) ? sizeof(INT_16) : sizeof(FLOAT_32));; + header.mSampleKind = targetKind; + + WriteHTKHeader (pOutFp, header, swap); + + if(targetKind & PARAMKIND_C) { + p_scale = (FLOAT*) malloc(n_coeffs * sizeof(FLOAT)); + p_bias = (FLOAT*) malloc(n_coeffs * sizeof(FLOAT)); + if (p_scale == NULL || p_bias == NULL) Error("Insufficient memory"); + + for(i = 0; i < n_coeffs; i++) { + float xmin, xmax; + xmin = xmax = rFeatureMatrix[0][i]; + + for(j = 1; j < n_samples; j++) { + if(rFeatureMatrix[j][i] > xmax) xmax = rFeatureMatrix[j][i]; + if(rFeatureMatrix[j][i] < xmin) xmin = rFeatureMatrix[j][i]; + } + + p_scale[i] = (2*32767) / (xmax - xmin); + p_bias[i] = p_scale[i] * (xmax + xmin) / 2; + } + + if (WriteHTKFeature(pOutFp, p_scale, n_coeffs, swap, false, 0, 0) + || WriteHTKFeature(pOutFp, p_bias, n_coeffs, swap, false, 0, 0)) { + return -1; + } + } + + for(j = 0; j < n_samples; j++) { + if (WriteHTKFeature(pOutFp, rFeatureMatrix[j].pData(), n_coeffs, swap, targetKind & PARAMKIND_C, p_scale, p_bias)) { + return -1; + } + } + + return 0; + } + + //*************************************************************************** + //*************************************************************************** + + + bool + FeatureRepository:: + ReadGzipAsciiFeatures(const FileListElem& rFileNameRecord, Matrix<BaseFloat>& rFeatureMatrix) + { + //build the command + std::string cmd("gunzip -c "); cmd += rFileNameRecord.Physical(); + + //define buffer + const int buf_size=262144; + char buf[buf_size]; + char vbuf[2*buf_size]; + + TIMER_START(mTim); + //open the pipe + FILE* fp = popen(cmd.c_str(),"r"); + if(fp == NULL) { + //2nd try... + Warning(std::string("2nd try to open pipe: ")+cmd); + sleep(5); + fp = popen(cmd.c_str(),"r"); + if(fp == NULL) { + KALDI_ERR << "Cannot open pipe: " << cmd; + } + } + setvbuf(fp,vbuf,_IOFBF,2*buf_size); + TIMER_END(mTim,mTimeOpen); + + //string will stay allocated across calls + static std::string line; line.resize(0); + + //define matrix storage + static int cols = 131072; + std::list<std::vector<BaseFloat> > matrix(1); + matrix.front().reserve(cols); + + //read all the lines to a vector + int line_ctr=1; + while(1) { + TIMER_START(mTim); + if(NULL == fgets(buf,buf_size,fp)) break; + TIMER_END(mTim,mTimeRead); + + line += buf; + if(*(line.rbegin()) == '\n' || feof(fp)) { + //parse the line of numbers + TIMER_START(mTim); + const char* ptr = line.c_str(); + char* end; + while(1) { + //skip whitespace + while(isspace(*ptr)) ptr++; + if(*ptr == 0) break; + //check that a number follows + switch(*ptr) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case '.': case '+': case '-': + break; + default : KALDI_ERR << "A number was expected:" << ptr + << " reading from" << cmd; + exit(1); + } + //read a number + BaseFloat val = strtof(ptr,&end); ptr=end; + matrix.back().push_back(val); + } + TIMER_END(mTim,mTimeNormalize); + //we have the line of numbers, insert empty row to matrix + if(matrix.back().size() > 0 && !feof(fp)) { + matrix.push_back(std::vector<BaseFloat>()); + matrix.back().reserve(matrix.front().size()); + } + //dispose the current line + line.resize(0);//but stay allocated... + line_ctr++; + } + } + if(matrix.back().size() == 0) matrix.pop_back(); + + //get matrix dimensions + int rows = matrix.size(); + /*int*/ cols = matrix.front().size(); + + //define interators + std::list<std::vector<BaseFloat> >::iterator it_r; + std::vector<BaseFloat>::iterator it_c; + + //check that all lines have same size + int i; + for(i=0,it_r=matrix.begin(); it_r != matrix.end(); ++i,++it_r) { + if(it_r->size() != cols) { + KALDI_ERR << "All rows must have same dimension, 1st line cols: " << cols + << ", " << i << "th line cols: " << it_r->size(); + } + } + + //copy data to matrix + TIMER_START(mTim); + rFeatureMatrix.Init(rows,cols); + int r,c; + for(r=0,it_r=matrix.begin(); it_r!=matrix.end(); ++r,++it_r) { + for(c=0,it_c=it_r->begin(); it_c!=it_r->end(); ++c,++it_c) { + rFeatureMatrix(r,c) = *it_c; + } + } + TIMER_END(mTim,mTimeSeek); + + //close the pipe + if(pclose(fp) == -1) { + KALDI_ERR << "Cannot close pipe: " << cmd; + } + + return true; + } + + + //*************************************************************************** + //*************************************************************************** + +} // namespace TNet diff --git a/tnet_io/KaldiLib/Features.h b/tnet_io/KaldiLib/Features.h new file mode 100644 index 0000000..0980ab6 --- /dev/null +++ b/tnet_io/KaldiLib/Features.h @@ -0,0 +1,597 @@ +// +// C++ Interface: %{MODULE} +// +// Description: +// +// +// Author: %{AUTHOR} <%{EMAIL}>, (C) %{YEAR} +// +// Copyright: See COPYING file that comes with this distribution +// +// + +#ifndef TNet_Features_h +#define TNet_Features_h + +//***************************************************************************** +//***************************************************************************** +// Standard includes +// +#include <list> +#include <queue> +#include <string> + + +//***************************************************************************** +//***************************************************************************** +// Specific includes +// +#include "Common.h" +#include "Matrix.h" +#include "StkStream.h" +#include "Types.h" +#include "Timer.h" + + + +// we need these for reading and writing +#define UINT_16 unsigned short +#define UINT_32 unsigned +#define INT_16 short +#define INT_32 int +#define FLOAT_32 float +#define DOUBLE_64 double + + +#define PARAMKIND_WAVEFORM 0 +#define PARAMKIND_LPC 1 +#define PARAMKIND_LPREFC 2 +#define PARAMKIND_LPCEPSTRA 3 +#define PARAMKIND_LPDELCEP 4 +#define PARAMKIND_IREFC 5 +#define PARAMKIND_MFCC 6 +#define PARAMKIND_FBANK 7 +#define PARAMKIND_MELSPEC 8 +#define PARAMKIND_USER 9 +#define PARAMKIND_DISCRETE 10 +#define PARAMKIND_PLP 11 +#define PARAMKIND_ANON 12 + +#define PARAMKIND_E 0000100 /// has energy +#define PARAMKIND_N 0000200 /// absolute energy suppressed +#define PARAMKIND_D 0000400 /// has delta coefficients +#define PARAMKIND_A 0001000 /// has acceleration coefficients +#define PARAMKIND_C 0002000 /// is compressed +#define PARAMKIND_Z 0004000 /// has zero mean static coef. +#define PARAMKIND_K 0010000 /// has CRC checksum +#define PARAMKIND_0 0020000 /// has 0'th cepstral coef. +#define PARAMKIND_V 0040000 /// has VQ codebook index +#define PARAMKIND_T 0100000 /// has triple delta coefficients + + +//***************************************************************************** +//***************************************************************************** +// Code ... +// + +namespace TNet +{ + + /** ************************************************************************** + ** ************************************************************************** + */ + class FileListElem + { + private: + std::string mLogical; ///< Logical file name representation + std::string mPhysical; ///< Pysical file name representation + float mWeight; + + public: + FileListElem(const std::string & rFileName); + ~FileListElem() {} + + const std::string & + Logical() const { return mLogical; } + + const std::string & + Physical() const { return mPhysical; } + + const float& + Weight() const { return mWeight; } + }; + + /** ************************************************************************* + * @brief + */ + class FeatureRepository + { + public: + /** + * @brief HTK parameter file header (see HTK manual) + */ + struct HtkHeader + { + int mNSamples; + int mSamplePeriod; + short mSampleSize; + short mSampleKind; + + HtkHeader() + : mNSamples(0),mSamplePeriod(100000),mSampleSize(0),mSampleKind(12) + { } + }; + + + /** + * @brief Extension of the HTK header + */ + struct HtkHeaderExt + { + int mHeaderSize; + int mVersion; + int mSampSize; + }; + + + /** + * @brief Normalization file type + */ + enum CNFileType + { + CNF_Mean, + CNF_Variance, + CNF_VarScale + }; + + + static int + ReadParmKind(const char *pStr, bool checkBrackets); + + static int + ParmKind2Str(unsigned parmKind, char *pOutstr); + + static void + ReadCepsNormFile( + const char* pFileName, + char** lastFile, + BaseFloat** vecBuff, + int sampleKind, + CNFileType type, + int coefs); + + static const char mpParmKindNames[13][16]; + + + + ////////////////////////////////////////////////////////////////////////////// + // PUBLIC SECTION + ////////////////////////////////////////////////////////////////////////////// + public: + /// Iterates through the list of feature file records + typedef std::list<FileListElem>::iterator ListIterator; + + // some params for loading features + bool mSwapFeatures; + int mStartFrameExt; + int mEndFrameExt; + int mTargetKind; + int mDerivOrder; + int* mDerivWinLengths; + const char* mpCvgFile; + //:TODO: get rid of these + const char* mpCmnPath; + const char* mpCmnMask; + const char* mpCvnPath; + const char* mpCvnMask; + + int mTrace; + + + // Constructors and destructors + /** + * @brief Default constructor that creates an empty repository + */ + FeatureRepository() : mDerivWinLengths(NULL), mpCvgFile(NULL), + mpCmnPath(NULL), mpCmnMask(NULL), mpCvnPath(NULL), mpCvnMask(NULL), + mTrace(0), + mpLastFileName(NULL), mLastFileName(""), mpLastCmnFile (NULL), + mpLastCvnFile (NULL), mpLastCvgFile (NULL), mpCmn(NULL), + mpCvn(NULL), mpCvg(NULL), mpA(NULL), mpB(NULL), + mTimeOpen(0), mTimeSeek(0), mTimeRead(0), mTimeNormalize(0) + { + mInputQueueIterator = mInputQueue.end(); + } + + /** + * @brief Copy constructor which copies filled repository + */ + FeatureRepository(const FeatureRepository& ori) + : mDerivWinLengths(NULL), mpCvgFile(NULL), + mpCmnPath(NULL), mpCmnMask(NULL), mpCvnPath(NULL), mpCvnMask(NULL), + mTrace(0), + mpLastFileName(NULL), mLastFileName(""), mpLastCmnFile (NULL), + mpLastCvnFile (NULL), mpLastCvgFile (NULL), mpCmn(NULL), + mpCvn(NULL), mpCvg(NULL), mpA(NULL), mpB(NULL), + mTimeOpen(0), mTimeSeek(0), mTimeRead(0), mTimeNormalize(0) + { + //copy all the data from the input queue + mInputQueue = ori.mInputQueue; + + //initialize like the original + Init( + ori.mSwapFeatures, + ori.mStartFrameExt, + ori.mEndFrameExt, + ori.mTargetKind, + ori.mDerivOrder, + ori.mDerivWinLengths, + ori.mpCmnPath, + ori.mpCmnMask, + ori.mpCvnPath, + ori.mpCvnMask, + ori.mpCvgFile); + + //set on the end + mInputQueueIterator = mInputQueue.end(); + //copy default header values + mHeader = ori.mHeader; + } + + + /** + * @brief Destroys the repository + */ + ~FeatureRepository() + { + if (NULL != mpA) { + free(mpA); + } + + if (NULL != mpB) { + free(mpB); + } + //remove all entries + mInputQueue.clear(); + + if(mTrace&4) { + std::cout << "[FeatureRepository -- open:" << mTimeOpen << "s seek:" << mTimeSeek << "s read:" << mTimeRead << "s normalize:" << mTimeNormalize << "s]\n"; + } + + } + + + /** + * @brief Initializes the object using the given parameters + * + * @param swap Boolean value specifies whether to swap bytes + * when reading file or not. + * @param extLeft Features read from file are extended with extLeft + * initial frames. Normally, these frames are + * repetitions of the first feature frame in file + * (with its derivative, if derivatives are preset in + * the file). However, if segment of feature frames + * is extracted according to range specification, the + * true feature frames from beyond the segment boundary + * are used, wherever it is possible. Note that value + * of extLeft can be also negative. In such case + * corresponding number of initial frames is discarded. + * @param extRight The paramerer is complementary to parameter extLeft + * and has obvious meaning. (Controls extensions over + * the last frame, last frame from file is repeated + * only if necessary). + * @param targetKind The parameters is used to check whether + * pHeader->mSampleKind match to requited targetKind + * and to control suppression of 0'th cepstral or + * energy coefficients accorging to modifiers _E, _0, + * and _N. Modifiers _D, _A and _T are ignored; + * Computation of derivatives is controled by parameters + * derivOrder and derivWinLen. Value PARAMKIND_ANON + * ensures that function do not result in targetKind + * mismatch error and cause no _E or _0 suppression. + * @param derivOrder Final features will be augmented with their + * derivatives up to 'derivOrder' order. If 'derivOrder' + * is negative value, no new derivatives are appended + * and derivatives that already present in feature file + * are preserved. Straight features are considered + * to be of zero order. If some derivatives are already + * present in feature file, these are not computed + * again, only higher order derivatives are appended + * if required. Note, that HTK feature file cannot + * contain higher order derivatives (e.g. double delta) + * without containing lower ones (e.g. delta). + * Derivative present in feature file that are of + * higher order than is required are discarded. + * Derivatives are computed in the final stage from + * (extracted segment of) feature frames possibly + * extended by repeated frames. Derivatives are + * computed using the same formula that is employed + * also by HTK tools. Lengths of windows used for + * computation of derivatives are passed in parameter + * derivWinLen. To compute derivatives for frames close + * to boundaries, frames before the first and after the + * last frame (of the extracted segment) are considered + * to be (yet another) repetitions of the first and the + * last frame, respectively. If the segment of frames + * is extracted according to range specification and + * parameters extLeft and extLeft are set to zero, the + * first and the last frames of the segment are + * considered to be repeated, eventough the true feature + * frames from beyond the segment boundary can be + * available in the file. Therefore, segment extracted + * from features that were before augmented with + * derivatives will differ + * from the same segment augmented with derivatives by + * this function. Difference will be of course only on + * boundaries and only in derivatives. This "incorrect" + * behavior was chosen to fully simulate behavior of + * HTK tools. To obtain more correct computation of + * derivatives, use parameters extLeft and extRight, + * which correctly extend segment with the true frames + * (if possible) and in resulting feature matrix ignore + * first extLeft and last extRight frames. For this + * purpose, both extLeft and extRight should be set to + * sum of all values in the array derivWinLen. + * @param pDerivWinLen Array of size derivOrder specifying lengths of + * windows used for computation of derivatives. + * Individual values represents one side context + * used in the computation. The each window length is + * therefore twice the value from array plus one. + * Value at index zero specify window length for first + * order derivatives (delta), higher indices + * corresponds to higher order derivatives. + * @param pCmnPath Cepstral mean normalization path + * @param pCmnMask Cepstral mean normalization mask + * @param pCvnPath Cepstral variance normalization path + * @param pCvnMask Cepstral variance normalization mask + * @param pCvgFile Global variance file to be parsed + * + * The given parameters are necessary for propper feature extraction + */ + void + Init( + bool swap, + int extLeft, + int extRight, + int targetKind, + int derivOrder, + int* pDerivWinLen, + const char* pCmnPath, + const char* pCmnMask, + const char* pCvnPath, + const char* pCvnMask, + const char* pCvgFile); + + + void Trace(int trace) + { mTrace = trace; } + + /** + * @brief Returns a refference to the current file header + */ + const HtkHeader& + CurrentHeader() const + { return mHeader; } + + /** + * @brief Returns a refference to the current file header + */ + const HtkHeaderExt& + CurrentHeaderExt() const + { return mHeaderExt; } + + /** + * @brief Returns the current file details + * + * @return Refference to a class @c FileListElem + * + * Logical and physical file names are stored in @c FileListElem class + */ + const std::list<FileListElem>::iterator& + pCurrentRecord() const + { return mInputQueueIterator; } + + + /** + * @brief Returns the following file details + * + * @return Refference to a class @c FileListElem + * + * Logical and physical file names are stored in @c FileListElem class + */ + const std::list<FileListElem>::iterator& + pFollowingRecord() const + { return mInputQueueIterator; } + + + void + Rewind() + { mInputQueueIterator = mInputQueue.begin(); } + + + /** + * @brief Adds a single feature file to the repository + * @param rFileName file to read features from + */ + void + AddFile(const std::string & rFileName); + + + /** + * @brief Adds a list of feature files to the repository + * @param rFileName feature list file to read from + */ + void + AddFileList(const char* pFileName, const char* pFilter = ""); + + + const FileListElem& + Current() const + { return *mInputQueueIterator; } + + + /** + * @brief Moves to the next record + */ + void + MoveNext(); + + /** + * @brief Reads full feature matrix from a feature file + * @param rMatrix matrix to be created and filled with read data + * @return number of successfully read feature vectors + */ + bool + ReadFullMatrix(Matrix<BaseFloat>& rMatrix); + + bool + WriteFeatureMatrix(const Matrix<BaseFloat>& rMatrix, const std::string& filename, int targetKind, int samplePeriod); + + size_t + QueueSize() const {return mInputQueue.size(); } + + /** + * @brief Reads feature vectors from a feature file + * @param rMatrix matrix to be (only!) filled with read data. + * @return number of successfully read feature vectors + * + * The function tries to fill @c pMatrix with feature vectors comming from + * the current stream. If there are less vectors left in the stream, + * they are used and true number of successfuly read vectors is returned. + */ + int + ReadPartialMatrix(Matrix<BaseFloat>& rMatrix); + + /** + * @brief Filters the records of this repository based on HTK logical name + * masking. If pFilter equals to NULL, all source repository entries are + * coppied to rOut repository. + * + * @param pFilter HTK mask that defines the filter + * @param pValue Filter value + * @param rOut Reference to the new FeatureRepository which will be filled + * with the matching records + */ + void + HtkFilter(const char* pFilter, const char* pValue, FeatureRepository& rOut); + + + /** + * @brief Filters the records of this repository based on HTK logical name + * masking and returns list of unique names. If pFilter equals to NULL, + * single name "default" is returned. + * + * @param pFilter HTK mask that defines the filter + * @param rOut Reference to the list of results (std::list< std::string >) + */ + void + HtkSelection(const char* pFilter, std::list< std::string >& rOut); + + + /** + * @brief Returns true if there are no feature files left on input + */ + bool + EndOfList() const + { return mInputQueueIterator == mInputQueue.end(); } + + const std::string& + CurrentIndexFileName() const + { return mCurrentIndexFileName; } + + friend + void + AddFileListToFeatureRepositories( + const char* pFileName, + const char* pFilter, + std::queue<FeatureRepository *> &featureRepositoryList); + + +//////////////////////////////////////////////////////////////////////////////// +// PRIVATE SECTION +//////////////////////////////////////////////////////////////////////////////// + private: + /// List (queue) of input feature files + std::list<FileListElem> mInputQueue; + std::list<FileListElem>::iterator mInputQueueIterator; + + std::string mCurrentIndexFileName; + std::string mCurrentIndexFileDir; + std::string mCurrentIndexFileExt; + + /// current stream + IStkStream mStream; + + // stores feature file's HTK header + HtkHeader mHeader; + HtkHeaderExt mHeaderExt; + + + // this group of variables serve for working withthe same physical + // file name more than once + char* mpLastFileName; + std::string mLastFileName; + char* mpLastCmnFile; + char* mpLastCvnFile; + char* mpLastCvgFile; + BaseFloat* mpCmn; + BaseFloat* mpCvn; + BaseFloat* mpCvg; + HtkHeader mLastHeader; + BaseFloat* mpA; + BaseFloat* mpB; + + + + Timer mTim; + double mTimeOpen; + double mTimeSeek; + double mTimeRead; + double mTimeNormalize; + + + // Reads HTK feature file header + int + ReadHTKHeader(); + + int + ReadHTKFeature(BaseFloat* pIn, + size_t feaLen, + bool decompress, + BaseFloat* pScale, + BaseFloat* pBias); + + + bool + ReadHTKFeatures(const std::string& rFileName, Matrix<BaseFloat>& rFeatureMatrix); + + bool + ReadHTKFeatures(const FileListElem& rFileNameRecord, Matrix<BaseFloat>& rFeatureMatrix); + + + int + WriteHTKHeader (FILE* fp_out, HtkHeader header, bool swap); + + int + WriteHTKFeature (FILE* fp_out, FLOAT *out, size_t fea_len, bool swap, bool compress, FLOAT* pScale, FLOAT* pBias); + + int + WriteHTKFeatures(FILE* pOutFp, FLOAT * pOut, int nCoeffs, int nSamples, int samplePeriod, int targetKind, bool swap); + + int + WriteHTKFeatures( + FILE * pOutFp, + int samplePeriod, + int targetKind, + bool swap, + Matrix<BaseFloat>& rFeatureMatrix + ); + + bool + ReadGzipAsciiFeatures(const FileListElem& rFileNameRecord, Matrix<BaseFloat>& rFeatureMatrix); + + }; // class FeatureStream + +} //namespace TNet + +#endif // TNet_Features_h diff --git a/tnet_io/KaldiLib/Labels.cc b/tnet_io/KaldiLib/Labels.cc new file mode 100644 index 0000000..1af4447 --- /dev/null +++ b/tnet_io/KaldiLib/Labels.cc @@ -0,0 +1,609 @@ +#include "Labels.h" +#include "Timer.h" +#include "Error.h" +#include <cstdio> +#include <sstream> + + +namespace TNet { + + + //////////////////////////////////////////////////////////////////////// + // Class LabelRepository:: + void + LabelRepository:: + Init(const char* pLabelMlfFile, const char* pOutputLabelMapFile, const char* pLabelDir, const char* pLabelExt) { + InitMap(pLabelMlfFile, pOutputLabelMapFile, pLabelDir, pLabelExt); + } + void + LabelRepository:: + InitExt(const char* pLabelMlfFile, const char* fmt, const char *arg, const char* pLabelDir, const char* pLabelExt) { + if (strcmp(fmt, "map") == 0) + { + InitMap(pLabelMlfFile, arg, pLabelDir, pLabelExt); + mlf_fmt = MAP; + } + else if (strcmp(fmt, "raw") == 0) + { + InitRaw(pLabelMlfFile, arg, pLabelDir, pLabelExt); + mlf_fmt = RAW; + } + } + + void + LabelRepository:: + InitMap(const char* pLabelMlfFile, const char* pOutputLabelMapFile, const char* pLabelDir, const char* pLabelExt) + { + assert(NULL != pLabelMlfFile); + assert(NULL != pOutputLabelMapFile); + + // initialize the label streams + delete mpLabelStream; //if NULL, does nothing + delete _mpLabelStream; + _mpLabelStream = new std::ifstream(pLabelMlfFile); + mpLabelStream = new IMlfStream(*_mpLabelStream); + + // Label stream is initialized, just test it + if(!mpLabelStream->good()) + Error(std::string("Cannot open Label MLF file: ")+pLabelMlfFile); + + // Index the labels (good for randomized file lists) + Timer tim; tim.Start(); + mpLabelStream->Index(); + tim.End(); mIndexTime += tim.Val(); + + // Read the state-label to state-id map + ReadOutputLabelMap(pOutputLabelMapFile); + + // Store the label dir/ext + mpLabelDir = pLabelDir; + mpLabelExt = pLabelExt; + } + + void + LabelRepository:: + InitRaw(const char* pLabelMlfFile, const char *arg, const char* pLabelDir, const char* pLabelExt) + { + assert(NULL != pLabelMlfFile); + std::istringstream iss(arg); + size_t dim; + iss >> dim; + if (iss.fail() || dim <= 0) + PError("[lab] malformed dimension specification"); + raw_dim = dim; + // initialize the label streams + delete mpLabelStream; //if NULL, does nothing + delete _mpLabelStream; + _mpLabelStream = new std::ifstream(pLabelMlfFile); + mpLabelStream = new IMlfStream(*_mpLabelStream); + + // Label stream is initialized, just test it + if(!mpLabelStream->good()) + Error(std::string("Cannot open Label MLF file: ")+pLabelMlfFile); + + // Index the labels (good for randomized file lists) + Timer tim; tim.Start(); + mpLabelStream->Index(); + tim.End(); mIndexTime += tim.Val(); + + // Read the state-label to state-id map + //ReadOutputLabelMap(pOutputLabelMapFile); + + // Store the label dir/ext + mpLabelDir = pLabelDir; + mpLabelExt = pLabelExt; + } + + + void + LabelRepository:: + GenDesiredMatrix(BfMatrix& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical, bool has_vad) + { + //timer + Timer tim; tim.Start(); + + //Get the MLF stream reference... + IMlfStream& mLabelStream = *mpLabelStream; + //Build the file name of the label + MakeHtkFileName(mpLabelFile, pFeatureLogical, mpLabelDir, mpLabelExt); + + //Find block in MLF file + mLabelStream.Open(mpLabelFile); + if(!mLabelStream.good()) { + Error(std::string("Cannot open label MLF record: ") + mpLabelFile); + } + + + //resize the matrix + if(nFrames < 1) { + KALDI_ERR << "Number of frames:" << nFrames << " is lower than 1!!!\n" + << pFeatureLogical; + } + int label_map_size = mLabelMap.size(); + rDesired.Init(nFrames, label_map_size + (has_vad ? 2 : 0), true); //true: Zero() + + //aux variables + std::string line, state; + unsigned long long beg, end; + size_t state_index; + size_t trunc_frames = 0; + TagToIdMap::iterator it; + int vad_state; + + //parse the label file + while(!mLabelStream.eof()) { + std::getline(mLabelStream, line); + if(line == "") continue; //skip newlines/comments from MLF + if(line[0] == '#') continue; + + std::istringstream& iss = mGenDesiredMatrixStream; + iss.clear(); + iss.str(line); + + //parse the line + //begin + iss >> std::ws >> beg; + if(iss.fail()) { + KALDI_ERR << "Cannot parse column 1 (begin)\n" + << "line: " << line << "\n" + << "file: " << mpLabelFile << "\n"; + } + //end + iss >> std::ws >> end; + if(iss.fail()) { + KALDI_ERR << "Cannot parse column 2 (end)\n" + << "line: " << line << "\n" + << "file: " << mpLabelFile << "\n"; + } + //state tag + iss >> std::ws >> state; + if(iss.fail()) { + KALDI_ERR << "Cannot parse column 3 (state_tag)\n" + << "line: " << line << "\n" + << "file: " << mpLabelFile << "\n"; + } + + if (has_vad) /* an additional column for vad */ + { + iss >> std::ws >> vad_state; + if(iss.fail()) { + KALDI_ERR << "Cannot parse column 4 (vad_state)\n" + << "line: " << line << "\n" + << "file: " << mpLabelFile << "\n"; + } + } + + //fprintf(stderr, "Parsed: %lld %lld %s\n", beg, end, state.c_str()); + + //divide beg/end by sourceRate and round up to get interval of frames + beg = (beg+sourceRate/2)/sourceRate; + end = (end+sourceRate/2)/sourceRate; + //beg = (int)round(beg / (double)sourceRate); + //end = (int)round(end / (double)sourceRate); + + //find the state id + it = mLabelMap.find(state); + if(mLabelMap.end() == it) { + Error(std::string("Unknown state tag: '") + state + "' file:'" + mpLabelFile); + } + state_index = it->second; + + // Fill the desired matrix + for(unsigned long long frame=beg; frame<end; frame++) { + //don't write after matrix... (possible longer transcript than feature file) + if(frame >= (int)rDesired.Rows()) { trunc_frames++; continue; } + + //check the next frame is empty: + if(0.0 != rDesired[frame].Sum()) { + //ERROR!!! + //find out what was previously filled!!! + BaseFloat max = rDesired[frame].Max(); + int idx = -1; + for(int i=0; i<(int)rDesired[frame].Dim(); i++) { + if(rDesired[frame][i] == max) idx = i; + } + for(it=mLabelMap.begin(); it!=mLabelMap.end(); ++it) { + if((int)it->second == idx) break; + } + std::string state_prev = "error"; + if(it != mLabelMap.end()) { + state_prev = it->first; + } + //print the error message + std::ostringstream os; + os << "Frame already assigned to other state, " + << " file: " << mpLabelFile + << " frame: " << frame + << " nframes: " << nFrames + << " sum: " << rDesired[frame].Sum() + << " previously assigned to: " << state_prev << "(" << idx << ")" + << " now should be assigned to: " << state << "(" << state_index << ")" + << "\n"; + Error(os.str()); + } + + //fill the row + rDesired[(size_t)frame][state_index] = 1.0f; + if (has_vad) + rDesired[(size_t)frame][label_map_size + !!vad_state] = 1.0f; + } + } + + mLabelStream.Close(); + + //check the desired matrix (rows sum up to 1.0) + for(size_t i=0; i<rDesired.Rows(); ++i) { + float desired_row_sum = rDesired[i].Sum(); + if(!desired_row_sum == 1.0) { + std::ostringstream os; + os << "Desired vector sum isn't 1.0, " + << " file: " << mpLabelFile + << " row: " << i + << " nframes: " << nFrames + << " content: " << rDesired[i] + << " sum: " << desired_row_sum << "\n"; + Error(os.str()); + } + } + + //warning when truncating many frames + if(trunc_frames > 10) { + std::ostringstream os; + os << "Truncated frames: " << trunc_frames + << " Check sourcerate in features and validity of labels\n"; + Warning(os.str()); + } + + //timer + tim.End(); mGenDesiredMatrixTime += tim.Val(); + } + + + + void + LabelRepository:: + ReadOutputLabelMap(const char* file) + { + assert(mLabelMap.size() == 0); + int i = 0; + std::string state_tag; + std::ifstream in(file); + if(!in.good()) + Error(std::string("Cannot open OutputLabelMapFile: ")+file); + + in >> std::ws; + while(!in.eof()) { + in >> state_tag; + in >> std::ws; + assert(mLabelMap.find(state_tag) == mLabelMap.end()); + mLabelMap[state_tag] = i++; + } + + in.close(); + assert(mLabelMap.size() > 0); + } + + + void + LabelRepository:: + GenDesiredMatrixExt(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical) { + switch (mlf_fmt) + { + case MAP: GenDesiredMatrixExtMap(rDesired, nFrames, sourceRate, pFeatureLogical); + break; + case RAW: GenDesiredMatrixExtRaw(rDesired, nFrames, sourceRate, pFeatureLogical); + break; + default: assert(0); + } + } + + void + LabelRepository:: + GenDesiredMatrixExtMap(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical) + { + //timer + Timer tim; tim.Start(); + + //Get the MLF stream reference... + IMlfStream& mLabelStream = *mpLabelStream; + //Build the file name of the label + MakeHtkFileName(mpLabelFile, pFeatureLogical, mpLabelDir, mpLabelExt); + + //Find block in MLF file + mLabelStream.Open(mpLabelFile); + if(!mLabelStream.good()) { + Error(std::string("Cannot open label MLF record: ") + mpLabelFile); + } + + + //resize the matrix + if(nFrames < 1) { + KALDI_ERR << "Number of frames:" << nFrames << " is lower than 1!!!\n" + << pFeatureLogical; + } + + size_t prev = rDesired.size(); + rDesired.resize(prev + 1, BfMatrix()); /* state + vad */ + int label_map_size = mLabelMap.size(); + rDesired[prev].Init(nFrames, 1, true); //true: Zero() + + //aux variables + std::string line, state; + unsigned long long beg, end; + size_t state_index; + size_t trunc_frames = 0; + TagToIdMap::iterator it; + + //parse the label file + while(!mLabelStream.eof()) { + std::getline(mLabelStream, line); + if(line == "") continue; //skip newlines/comments from MLF + if(line[0] == '#') continue; + + std::istringstream& iss = mGenDesiredMatrixStream; + iss.clear(); + iss.str(line); + + //parse the line + //begin + iss >> std::ws >> beg; + if(iss.fail()) { + KALDI_ERR << "Cannot parse column 1 (begin)\n" + << "line: " << line << "\n" + << "file: " << mpLabelFile << "\n"; + } + //end + iss >> std::ws >> end; + if(iss.fail()) { + KALDI_ERR << "Cannot parse column 2 (end)\n" + << "line: " << line << "\n" + << "file: " << mpLabelFile << "\n"; + } + //state tag + iss >> std::ws >> state; + if(iss.fail()) { + KALDI_ERR << "Cannot parse column 3 (state_tag)\n" + << "line: " << line << "\n" + << "file: " << mpLabelFile << "\n"; + } + + + //fprintf(stderr, "Parsed: %lld %lld %s\n", beg, end, state.c_str()); + + //divide beg/end by sourceRate and round up to get interval of frames + beg = (beg+sourceRate/2)/sourceRate; + end = (end+sourceRate/2)/sourceRate; + //beg = (int)round(beg / (double)sourceRate); + //end = (int)round(end / (double)sourceRate); + + //find the state id + it = mLabelMap.find(state); + if(mLabelMap.end() == it) { + Error(std::string("Unknown state tag: '") + state + "' file:'" + mpLabelFile); + } + state_index = it->second; + + // Fill the desired matrix + for(unsigned long long frame=beg; frame<end; frame++) { + //don't write after matrix... (possible longer transcript than feature file) + if(frame >= (int)rDesired[prev].Rows()) { trunc_frames++; continue; } + + //check the next frame is empty: + if(0.0 != rDesired[prev][frame][0]) { + //ERROR!!! + //find out what was previously filled!!! + /* + BaseFloat max = rDesired[prev][frame].Max(); + int idx = -1; + for(int i=0; i<(int)rDesired[prev][frame].Dim(); i++) { + if(rDesired[prev][frame][i] == max) idx = i; + } + */ + BaseFloat max = rDesired[prev][frame][0]; + int idx = round(max); + for(it=mLabelMap.begin(); it!=mLabelMap.end(); ++it) { + if((int)it->second == idx) break; + } + std::string state_prev = "error"; + if(it != mLabelMap.end()) { + state_prev = it->first; + } + //print the error message + std::ostringstream os; + os << "Frame already assigned to other state, " + << " file: " << mpLabelFile + << " frame: " << frame + << " nframes: " << nFrames + << " sum: " << max + << " previously assigned to: " << state_prev << "(" << idx << ")" + << " now should be assigned to: " << state << "(" << state_index << ")" + << "\n"; + Error(os.str()); + } + + //fill the row + //rDesired[prev][(size_t)frame][state_index] = 1.0f; + rDesired[prev][(size_t)frame][0] = state_index; + } + } + + mLabelStream.Close(); + /* + //check the desired matrix (rows sum up to 1.0) + for(size_t i=0; i<rDesired[prev].Rows(); ++i) { + float desired_row_sum = rDesired[prev][i].Sum(); + if(!desired_row_sum == 1.0) { + std::ostringstream os; + os << "Desired vector sum isn't 1.0, " + << " file: " << mpLabelFile + << " row: " << i + << " nframes: " << nFrames + << " content: " << rDesired[prev][i] + << " sum: " << desired_row_sum << "\n"; + Error(os.str()); + } + } + */ + + //warning when truncating many frames + if(trunc_frames > 10) { + std::ostringstream os; + os << "Truncated frames: " << trunc_frames + << " Check sourcerate in features and validity of labels\n"; + Warning(os.str()); + } + + //timer + tim.End(); mGenDesiredMatrixTime += tim.Val(); + } + + void + LabelRepository:: + GenDesiredMatrixExtRaw(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical) + { + //timer + Timer tim; tim.Start(); + + //Get the MLF stream reference... + IMlfStream& mLabelStream = *mpLabelStream; + //Build the file name of the label + MakeHtkFileName(mpLabelFile, pFeatureLogical, mpLabelDir, mpLabelExt); + + //Find block in MLF file + mLabelStream.Open(mpLabelFile); + if(!mLabelStream.good()) { + Error(std::string("Cannot open label MLF record: ") + mpLabelFile); + } + + + //resize the matrix + if(nFrames < 1) { + KALDI_ERR << "Number of frames:" << nFrames << " is lower than 1!!!\n" + << pFeatureLogical; + } + + size_t prev = rDesired.size(); + rDesired.resize(prev + 1, BfMatrix()); /* state + vad */ + rDesired[prev].Init(nFrames, raw_dim, true); //true: Zero() + + //aux variables + std::string line, state; + unsigned long long beg, end; + size_t trunc_frames = 0; + Vector<BaseFloat> raw; + raw.Init(raw_dim); + + //parse the label file + while(!mLabelStream.eof()) { + std::getline(mLabelStream, line); + if(line == "") continue; //skip newlines/comments from MLF + if(line[0] == '#') continue; + + std::istringstream& iss = mGenDesiredMatrixStream; + iss.clear(); + iss.str(line); + + //parse the line + //begin + iss >> std::ws >> beg; + if(iss.fail()) { + KALDI_ERR << "Cannot parse column 1 (begin)\n" + << "line: " << line << "\n" + << "file: " << mpLabelFile << "\n"; + } + //end + iss >> std::ws >> end; + if(iss.fail()) { + KALDI_ERR << "Cannot parse column 2 (end)\n" + << "line: " << line << "\n" + << "file: " << mpLabelFile << "\n"; + } + + for (size_t i = 0; i < raw_dim; i++) + { + if (iss.eof()) + PError("[label] insufficient columns for the label: %s", mpLabelFile); + iss >> raw[i]; + if (iss.fail()) + PError("[label] cannot parse raw value for the label: %s", mpLabelFile); + } + /* + for (size_t i = 0; i < raw_dim; i++) + fprintf(stderr, "%.3f", raw[i]); + fprintf(stderr, "\n"); + */ + //divide beg/end by sourceRate and round up to get interval of frames + beg = (beg+sourceRate/2)/sourceRate; + if (end == (unsigned long long)-1) + end = rDesired[prev].Rows(); + else + end = (end+sourceRate/2)/sourceRate; + //printf("end:%lld\n", end); + //beg = (int)round(beg / (double)sourceRate); + //end = (int)round(end / (double)sourceRate); + + // Fill the desired matrix + for(unsigned long long frame=beg; frame<end; frame++) { + //don't write after matrix... (possible longer transcript than feature file) + if(frame >= (int)rDesired[prev].Rows()) { trunc_frames++; continue; } + + //check the next frame is empty: + if(0.0 != rDesired[prev][frame][0]) { + //ERROR!!! + //find out what was previously filled!!! + /* + BaseFloat max = rDesired[prev][frame].Max(); + int idx = -1; + for(int i=0; i<(int)rDesired[prev][frame].Dim(); i++) { + if(rDesired[prev][frame][i] == max) idx = i; + } + */ + BaseFloat max = rDesired[prev][frame][0]; + //print the error message + std::ostringstream os; + os << "Frame already assigned to other state, " + << " file: " << mpLabelFile + << " frame: " << frame + << " nframes: " << nFrames + << " sum: " << max + << "\n"; + Error(os.str()); + } + + //fill the row + //rDesired[prev][(size_t)frame][state_index] = 1.0f; + rDesired[prev][(size_t)frame].Copy(raw); + } + } + + mLabelStream.Close(); + /* + //check the desired matrix (rows sum up to 1.0) + for(size_t i=0; i<rDesired[prev].Rows(); ++i) { + float desired_row_sum = rDesired[prev][i].Sum(); + if(!desired_row_sum == 1.0) { + std::ostringstream os; + os << "Desired vector sum isn't 1.0, " + << " file: " << mpLabelFile + << " row: " << i + << " nframes: " << nFrames + << " content: " << rDesired[prev][i] + << " sum: " << desired_row_sum << "\n"; + Error(os.str()); + } + } + */ + + //warning when truncating many frames + if(trunc_frames > 10) { + std::ostringstream os; + os << "Truncated frames: " << trunc_frames + << " Check sourcerate in features and validity of labels\n"; + Warning(os.str()); + } + + //timer + tim.End(); mGenDesiredMatrixTime += tim.Val(); + } + +}//namespace diff --git a/tnet_io/KaldiLib/Labels.h b/tnet_io/KaldiLib/Labels.h new file mode 100644 index 0000000..409a080 --- /dev/null +++ b/tnet_io/KaldiLib/Labels.h @@ -0,0 +1,90 @@ +#ifndef _LABELS_H_ +#define _LABELS_H_ + + +#include "Matrix.h" +#include "MlfStream.h" +#include "Features.h" + +#include <map> +#include <iostream> + +namespace TNet { + + + class FeaCatPool; + + /** + * Desired matrix generation object, + * supports background-reading and caching, however can be + * used in foreground as well by GenDesiredMatrix() + */ + class LabelRepository + { + typedef std::map<std::string,size_t> TagToIdMap; + + public: + enum MFormat { + MAP, + RAW + }; + + LabelRepository() + : _mpLabelStream(NULL), mpLabelStream(NULL), mpLabelDir(NULL), mpLabelExt(NULL), mGenDesiredMatrixTime(0), mIndexTime(0), mTrace(0) + { } + + ~LabelRepository() + { + if(mTrace&4) { + std::cout << "[LabelRepository -- indexing:" << mIndexTime << "s" + " genDesiredMatrix:" << mGenDesiredMatrixTime << "s]" << std::endl; + } + delete mpLabelStream; + delete _mpLabelStream; + } + + /// Initialize the LabelRepository + void Init(const char* pLabelMlfFile, const char* pOutputLabelMapFile, const char* pLabelDir, const char* pLabelExt); + void InitExt(const char* pLabelMlfFile, const char* fmt, const char* arg, const char* pLabelDir, const char* pLabelExt); + void InitMap(const char* pLabelMlfFile, const char* pOutputLabelMapFile, const char* pLabelDir, const char* pLabelExt); + void InitRaw(const char* pLabelMlfFile,const char* arg, const char* pLabelDir, const char* pLabelExt); + + /// Set trace level + void Trace(int trace) + { mTrace = trace; } + + /// Get desired matrix from labels + void GenDesiredMatrix(BfMatrix& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical, bool has_vad = false); + + void GenDesiredMatrixExt(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical); + void GenDesiredMatrixExtMap(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical); + void GenDesiredMatrixExtRaw(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical); + size_t getWidth() { return mLabelMap.size(); } + MFormat getFormat() { return mlf_fmt; } + private: + /// Prepare the state-label to state-id map + void ReadOutputLabelMap(const char* file); + + private: + // Streams and state-map + std::ifstream* _mpLabelStream; ///< Helper stream for Label stream + IMlfStream* mpLabelStream; ///< Label stream + std::istringstream mGenDesiredMatrixStream; ///< Label file parsing stream + + const char* mpLabelDir; ///< Label dir in MLF + const char* mpLabelExt; ///< Label ext in MLF + char mpLabelFile[4096]; ///< Buffer for filenames in MLF + + TagToIdMap mLabelMap; ///< Map of state tags to net output indices + + double mGenDesiredMatrixTime; + float mIndexTime; + + int mTrace; + MFormat mlf_fmt; + size_t raw_dim; + }; + +}//namespace + +#endif diff --git a/tnet_io/KaldiLib/Makefile b/tnet_io/KaldiLib/Makefile new file mode 100644 index 0000000..61e0a59 --- /dev/null +++ b/tnet_io/KaldiLib/Makefile @@ -0,0 +1,28 @@ + +include ../tnet.mk + +INCLUDE = -I. + +all: $(OBJ_DIR)/libKaldiLib.a + +$(OBJ_DIR)/libKaldiLib.a: $(OBJ) + $(AR) ruv $@ $(OBJ) + $(RANLIB) $@ + +$(OBJ_DIR)/%.o : %.cc + $(CXX) -o $@ -c $< $(CFLAGS) $(CXXFLAGS) $(INCLUDE) + + + +.PHONY: clean doc depend +clean: + rm -f $(OBJ_DIR)/*.o $(OBJ_DIR)/*.a + +doc: + doxygen ../../doc/doxyfile_TNetLib + +depend: + $(CXX) -M $(CXXFLAGS) *.cc $(INCLUDE) > .depend.mk + +-include .depend.mk + diff --git a/tnet_io/KaldiLib/MathAux.h b/tnet_io/KaldiLib/MathAux.h new file mode 100644 index 0000000..c08e836 --- /dev/null +++ b/tnet_io/KaldiLib/MathAux.h @@ -0,0 +1,117 @@ +#ifndef TNet_MathAux_h +#define TNet_MathAux_h + +#include <cmath> + + +#if !defined(SQR) +# define SQR(x) ((x) * (x)) +#endif + + +#if !defined(LOG_0) +# define LOG_0 (-1.0e10) +#endif + +#if !defined(LOG_MIN) +# define LOG_MIN (0.5 * LOG_0) +#endif + + +#ifndef DBL_EPSILON +#define DBL_EPSILON 2.2204460492503131e-16 +#endif + + +#ifndef M_PI +# define M_PI 3.1415926535897932384626433832795 +#endif + +#define M_LOG_2PI 1.8378770664093454835606594728112 + + +#if DOUBLEPRECISION +# define FLOAT double +# define EPSILON DBL_EPSILON +# define FLOAT_FMT "%lg" +# define swapFLOAT swap8 +# define _ABS fabs +# define _COS cos +# define _EXP exp +# define _LOG log +# define _SQRT sqrt +#else +# define FLOAT float +# define EPSILON FLT_EPSILON +# define FLOAT_FMT "%g" +# define swapFLOAT swap4 +# define _ABS fabsf +# define _COS cosf +# define _EXP expf +# define _LOG logf +# define _SQRT sqrtf +#endif + +namespace TNet +{ + inline float frand(){ // random between 0 and 1. + return (float(rand()) + 1.0f) / (float(RAND_MAX)+2.0f); + } + inline float gauss_rand(){ + return _SQRT( -2.0f * _LOG(frand()) ) * _COS(2.0f*float(M_PI)*frand()); + } + + static const double gMinLogDiff = log(DBL_EPSILON); + + //*************************************************************************** + //*************************************************************************** + inline double + LogAdd(double x, double y) + { + double diff; + + if (x < y) { + diff = x - y; + x = y; + } else { + diff = y - x; + } + + double res; + if (x >= LOG_MIN) { + if (diff >= gMinLogDiff) { + res = x + log(1.0 + exp(diff)); + } else { + res = x; + } + } else { + res = LOG_0; + } + return res; + } + + + //*************************************************************************** + //*************************************************************************** + inline double + LogSub(double x, double y) // returns exp(x) - exp(y). Throws exception if y>=x. + { + + if(y >= x){ + if(y==x) return LOG_0; + else throw std::runtime_error("LogSub: cannot subtract a larger from a smaller number."); + } + + double diff = y - x; // Will be negative. + + double res = x + log(1.0 - exp(diff)); + + if(res != res) // test for res==NaN.. could happen if diff ~0.0, so 1.0-exp(diff) == 0.0 to machine precision. + res = LOG_0; + return res; + } + +} // namespace TNet + + +#endif diff --git a/tnet_io/KaldiLib/Matrix.cc b/tnet_io/KaldiLib/Matrix.cc new file mode 100644 index 0000000..f9d5909 --- /dev/null +++ b/tnet_io/KaldiLib/Matrix.cc @@ -0,0 +1,295 @@ +/** + * @file Matrix.cc + * + * Implementation of specialized Matrix template methods + */ + + +#include "Matrix.h" + +#if defined(HAVE_CLAPACK) +#include "CLAPACK-3.1.1.1/INCLUDE/f2c.h" +extern "C" { +#include "CLAPACK-3.1.1.1/INCLUDE/clapack.h" +} +// These are some stupid clapack things that we want to get rid of +#ifdef min +#undef min +#endif + +#ifdef max +#undef max +#endif + +#endif + + + + +namespace TNet +{ + //*************************************************************************** + //*************************************************************************** +#ifdef HAVE_ATLAS + //*************************************************************************** + //*************************************************************************** + template<> + Matrix<float> & + Matrix<float>:: + Invert(float *LogDet, float *DetSign, bool inverse_needed) + { + assert(Rows() == Cols()); + +#if defined(HAVE_CLAPACK) + integer* pivot = new integer[mMRows]; + integer M = Rows(); + integer N = Cols(); + integer LDA = mStride; + integer result; + integer l_work = std::max<integer>(1, N); + float* p_work = new float[l_work]; + + sgetrf_(&M, &N, mpData, &LDA, pivot, &result); + const int pivot_offset=1; +#else + int* pivot = new int[mMRows]; + int result = clapack_sgetrf(CblasColMajor, Rows(), Cols(), mpData, mStride, pivot); + const int pivot_offset=0; +#endif + assert(result >= 0 && "Call to CLAPACK sgetrf_ or ATLAS clapack_sgetrf called with wrong arguments"); + if(result != 0) { + Error("Matrix is singular"); + } + if(DetSign!=NULL){ *DetSign=1.0; for(size_t i=0;i<mMRows;i++) if(pivot[i]!=(int)i+pivot_offset) *DetSign *= -1.0; } + if(LogDet!=NULL||DetSign!=NULL){ // Compute log determinant... + assert(mMRows==mMCols); // Can't take determinant of non-square matrix. + *LogDet = 0.0; float prod = 1.0; + for(size_t i=0;i<mMRows;i++){ + prod *= (*this)(i,i); + if(i==mMRows-1 || fabs(prod)<1.0e-10 || fabs(prod)>1.0e+10){ + if(LogDet!=NULL) *LogDet += log(fabs(prod)); + if(DetSign!=NULL) *DetSign *= (prod>0?1.0:-1.0); + prod=1.0; + } + } + } +#if defined(HAVE_CLAPACK) + if(inverse_needed) sgetri_(&M, mpData, &LDA, pivot, p_work, &l_work, &result); + delete [] pivot; +#else + if(inverse_needed) result = clapack_sgetri(CblasColMajor, Rows(), mpData, mStride, pivot); + delete [] pivot; +#endif + assert(result == 0 && "Call to CLAPACK sgetri_ or ATLAS clapack_sgetri called with wrong arguments"); + return *this; + } + + + //*************************************************************************** + //*************************************************************************** + template<> + Matrix<double> & + Matrix<double>:: + Invert(double *LogDet, double *DetSign, bool inverse_needed) + { + assert(Rows() == Cols()); + +#if defined(HAVE_CLAPACK) + integer* pivot = new integer[mMRows]; + integer M = Rows(); + integer N = Cols(); + integer LDA = mStride; + integer result; + integer l_work = std::max<integer>(1, N); + double* p_work = new double[l_work]; + + dgetrf_(&M, &N, mpData, &LDA, pivot, &result); + const int pivot_offset=1; +#else + int* pivot = new int[mMRows]; + int result = clapack_dgetrf(CblasColMajor, Rows(), Cols(), mpData, mStride, pivot); + const int pivot_offset=0; +#endif + assert(result >= 0 && "Call to CLAPACK dgetrf_ or ATLAS clapack_dgetrf called with wrong arguments"); + if(result != 0) { + Error("Matrix is singular"); + } + if(DetSign!=NULL){ *DetSign=1.0; for(size_t i=0;i<mMRows;i++) if(pivot[i]!=(int)i+pivot_offset) *DetSign *= -1.0; } + if(LogDet!=NULL||DetSign!=NULL){ // Compute log determinant... + assert(mMRows==mMCols); // Can't take determinant of non-square matrix. + *LogDet = 0.0; double prod = 1.0; + for(size_t i=0;i<mMRows;i++){ + prod *= (*this)(i,i); + if(i==mMRows-1 || fabs(prod)<1.0e-10 || fabs(prod)>1.0e+10){ + if(LogDet!=NULL) *LogDet += log(fabs(prod)); + if(DetSign!=NULL) *DetSign *= (prod>0?1.0:-1.0); + prod=1.0; + } + } + } +#if defined(HAVE_CLAPACK) + if(inverse_needed) dgetri_(&M, mpData, &LDA, pivot, p_work, &l_work, &result); + delete [] pivot; +#else + if(inverse_needed) result = clapack_dgetri(CblasColMajor, Rows(), mpData, mStride, pivot); + delete [] pivot; +#endif + assert(result == 0 && "Call to CLAPACK dgetri_ or ATLAS clapack_dgetri called with wrong arguments"); + return *this; + } + + template<> + Matrix<float> & + Matrix<float>:: + BlasGer(const float alpha, const Vector<float>& rA, const Vector<float>& rB) + { + assert(rA.Dim() == mMRows && rB.Dim() == mMCols); + cblas_sger(CblasRowMajor, rA.Dim(), rB.Dim(), alpha, rA.pData(), 1, rB.pData(), 1, mpData, mStride); + return *this; + } + + template<> + Matrix<double> & + Matrix<double>:: + BlasGer(const double alpha, const Vector<double>& rA, const Vector<double>& rB) + { + assert(rA.Dim() == mMRows && rB.Dim() == mMCols); + cblas_dger(CblasRowMajor, rA.Dim(), rB.Dim(), alpha, rA.pData(), 1, rB.pData(), 1, mpData, mStride); + return *this; + } + + template<> + Matrix<float>& + Matrix<float>:: + BlasGemm(const float alpha, + const Matrix<float>& rA, MatrixTrasposeType transA, + const Matrix<float>& rB, MatrixTrasposeType transB, + const float beta) + { + assert((transA == NO_TRANS && transB == NO_TRANS && rA.Cols() == rB.Rows() && rA.Rows() == Rows() && rB.Cols() == Cols()) + || (transA == TRANS && transB == NO_TRANS && rA.Rows() == rB.Rows() && rA.Cols() == Rows() && rB.Cols() == Cols()) + || (transA == NO_TRANS && transB == TRANS && rA.Cols() == rB.Cols() && rA.Rows() == Rows() && rB.Rows() == Cols()) + || (transA == TRANS && transB == TRANS && rA.Rows() == rB.Cols() && rA.Cols() == Rows() && rB.Rows() == Cols())); + + cblas_sgemm(CblasRowMajor, static_cast<CBLAS_TRANSPOSE>(transA), static_cast<CBLAS_TRANSPOSE>(transB), + Rows(), Cols(), transA == NO_TRANS ? rA.Cols() : rA.Rows(), + alpha, rA.mpData, rA.mStride, rB.mpData, rB.mStride, + beta, mpData, mStride); + return *this; + } + + template<> + Matrix<double>& + Matrix<double>:: + BlasGemm(const double alpha, + const Matrix<double>& rA, MatrixTrasposeType transA, + const Matrix<double>& rB, MatrixTrasposeType transB, + const double beta) + { + assert((transA == NO_TRANS && transB == NO_TRANS && rA.Cols() == rB.Rows() && rA.Rows() == Rows() && rB.Cols() == Cols()) + || (transA == TRANS && transB == NO_TRANS && rA.Rows() == rB.Rows() && rA.Cols() == Rows() && rB.Cols() == Cols()) + || (transA == NO_TRANS && transB == TRANS && rA.Cols() == rB.Cols() && rA.Rows() == Rows() && rB.Rows() == Cols()) + || (transA == TRANS && transB == TRANS && rA.Rows() == rB.Cols() && rA.Cols() == Rows() && rB.Rows() == Cols())); + + cblas_dgemm(CblasRowMajor, static_cast<CBLAS_TRANSPOSE>(transA), static_cast<CBLAS_TRANSPOSE>(transB), + Rows(), Cols(), transA == NO_TRANS ? rA.Cols() : rA.Rows(), + alpha, rA.mpData, rA.mStride, rB.mpData, rB.mStride, + beta, mpData, mStride); + return *this; + } + + template<> + Matrix<float>& + Matrix<float>:: + Axpy(const float alpha, + const Matrix<float>& rA, MatrixTrasposeType transA){ + int aStride = (int)rA.mStride, stride = mStride; + float *adata=rA.mpData, *data=mpData; + if(transA == NO_TRANS){ + assert(rA.Rows()==Rows() && rA.Cols()==Cols()); + for(size_t row=0;row<mMRows;row++,adata+=aStride,data+=stride) + cblas_saxpy(mMCols, alpha, adata, 1, data, 1); + } else { + assert(rA.Cols()==Rows() && rA.Rows()==Cols()); + for(size_t row=0;row<mMRows;row++,adata++,data+=stride) + cblas_saxpy(mMCols, alpha, adata, aStride, data, 1); + } + return *this; + } + + template<> + Matrix<double>& + Matrix<double>:: + Axpy(const double alpha, + const Matrix<double>& rA, MatrixTrasposeType transA){ + int aStride = (int)rA.mStride, stride = mStride; + double *adata=rA.mpData, *data=mpData; + if(transA == NO_TRANS){ + assert(rA.Rows()==Rows() && rA.Cols()==Cols()); + for(size_t row=0;row<mMRows;row++,adata+=aStride,data+=stride) + cblas_daxpy(mMCols, alpha, adata, 1, data, 1); + } else { + assert(rA.Cols()==Rows() && rA.Rows()==Cols()); + for(size_t row=0;row<mMRows;row++,adata++,data+=stride) + cblas_daxpy(mMCols, alpha, adata, aStride, data, 1); + } + return *this; + } + + template <> //non-member but friend! + double TraceOfProduct(const Matrix<double> &A, const Matrix<double> &B){ // tr(A B), equivalent to sum of each element of A times same element in B' + size_t aStride = A.mStride, bStride = B.mStride; + assert(A.Rows()==B.Cols() && A.Cols()==B.Rows()); + double ans = 0.0; + double *adata=A.mpData, *bdata=B.mpData; + size_t arows=A.Rows(), acols=A.Cols(); + for(size_t row=0;row<arows;row++,adata+=aStride,bdata++) + ans += cblas_ddot(acols, adata, 1, bdata, bStride); + return ans; + } + + template <> //non-member but friend! + double TraceOfProductT(const Matrix<double> &A, const Matrix<double> &B){ // tr(A B), equivalent to sum of each element of A times same element in B' + size_t aStride = A.mStride, bStride = B.mStride; + assert(A.Rows()==B.Rows() && A.Cols()==B.Cols()); + double ans = 0.0; + double *adata=A.mpData, *bdata=B.mpData; + size_t arows=A.Rows(), acols=A.Cols(); + for(size_t row=0;row<arows;row++,adata+=aStride,bdata+=bStride) + ans += cblas_ddot(acols, adata, 1, bdata, 1); + return ans; + } + + + template <> //non-member but friend! + float TraceOfProduct(const Matrix<float> &A, const Matrix<float> &B){ // tr(A B), equivalent to sum of each element of A times same element in B' + size_t aStride = A.mStride, bStride = B.mStride; + assert(A.Rows()==B.Cols() && A.Cols()==B.Rows()); + float ans = 0.0; + float *adata=A.mpData, *bdata=B.mpData; + size_t arows=A.Rows(), acols=A.Cols(); + for(size_t row=0;row<arows;row++,adata+=aStride,bdata++) + ans += cblas_sdot(acols, adata, 1, bdata, bStride); + return ans; + } + + template <> //non-member but friend! + float TraceOfProductT(const Matrix<float> &A, const Matrix<float> &B){ // tr(A B), equivalent to sum of each element of A times same element in B' + size_t aStride = A.mStride, bStride = B.mStride; + assert(A.Rows()==B.Rows() && A.Cols()==B.Cols()); + float ans = 0.0; + float *adata=A.mpData, *bdata=B.mpData; + size_t arows=A.Rows(), acols=A.Cols(); + for(size_t row=0;row<arows;row++,adata+=aStride,bdata+=bStride) + ans += cblas_sdot(acols, adata, 1, bdata, 1); + return ans; + } + + + + +#endif //HAVE_ATLAS + + + +} //namespace STK diff --git a/tnet_io/KaldiLib/Matrix.h b/tnet_io/KaldiLib/Matrix.h new file mode 100644 index 0000000..d33cb0c --- /dev/null +++ b/tnet_io/KaldiLib/Matrix.h @@ -0,0 +1,677 @@ +#ifndef TNet_Matrix_h +#define TNet_Matrix_h + +#include <stddef.h> +#include <stdlib.h> +#include <stdexcept> +#include <iostream> + +#ifdef HAVE_ATLAS +extern "C"{ + #include <cblas.h> + #include <clapack.h> +} +#endif + +#include "Common.h" +#include "MathAux.h" +#include "Types.h" +#include "Error.h" + +//#define TRACE_MATRIX_OPERATIONS +#define CHECKSIZE + +namespace TNet +{ + + + // class matrix_error : public std::logic_error {}; + // class matrix_sizes_error : public matrix_error {}; + + // declare the class so the header knows about it + template<typename _ElemT> class Vector; + template<typename _ElemT> class SubVector; + template<typename _ElemT> class Matrix; + template<typename _ElemT> class SubMatrix; + + // we need to declare the friend << operator here + template<typename _ElemT> + std::ostream & operator << (std::ostream & rOut, const Matrix<_ElemT> & rM); + + // we need to declare the friend << operator here + template<typename _ElemT> + std::istream & operator >> (std::istream & rIn, Matrix<_ElemT> & rM); + + // we need to declare this friend function here + template<typename _ElemT> + _ElemT TraceOfProduct(const Matrix<_ElemT> &A, const Matrix<_ElemT> &B); // tr(A B) + + // we need to declare this friend function here + template<typename _ElemT> + _ElemT TraceOfProductT(const Matrix<_ElemT> &A, const Matrix<_ElemT> &B); // tr(A B^T)==tr(A^T B) + + + /** ************************************************************************** + ** ************************************************************************** + * @brief Provides a matrix class + * + * This class provides a way to work with matrices in TNet. + * It encapsulates basic operations and memory optimizations. + * + */ + template<typename _ElemT> + class Matrix + { + public: + /// defines a transpose type + + struct HtkHeader + { + INT_32 mNSamples; + INT_32 mSamplePeriod; + INT_16 mSampleSize; + UINT_16 mSampleKind; + }; + + + /** + * @brief Extension of the HTK header + */ + struct HtkHeaderExt + { + INT_32 mHeaderSize; + INT_32 mVersion; + INT_32 mSampSize; + }; + + + + + /// defines a type of this + typedef Matrix<_ElemT> ThisType; + + // Constructors + + /// Empty constructor + Matrix<_ElemT> (): + mpData(NULL), mMCols(0), mMRows(0), mStride(0) +#ifdef STK_MEMALIGN_MANUAL + , mpFreeData(NULL) +#endif + {} + + /// Copy constructor + Matrix<_ElemT> (const Matrix<_ElemT> & rM, MatrixTrasposeType trans=NO_TRANS): + mpData(NULL) + { if(trans==NO_TRANS){ Init(rM.mMRows, rM.mMCols); Copy(rM); } else { Init(rM.mMCols,rM.mMRows); Copy(rM,TRANS); } } + + /// Copy constructor from another type. + template<typename _ElemU> + explicit Matrix<_ElemT> (const Matrix<_ElemU> & rM, MatrixTrasposeType trans=NO_TRANS): + mpData(NULL) + { if(trans==NO_TRANS){ Init(rM.Rows(), rM.Cols()); Copy(rM); } else { Init(rM.Cols(),rM.Rows()); Copy(rM,TRANS); } } + + /// Basic constructor + Matrix(const size_t r, const size_t c, bool clear=true) + { mpData=NULL; Init(r, c, clear); } + + + Matrix<_ElemT> &operator = (const Matrix <_ElemT> &other) { Init(other.Rows(), other.Cols()); Copy(other); return *this; } // Needed for inclusion in std::vector + + /// Destructor + ~Matrix() + { Destroy(); } + + + /// Initializes matrix (if not done by constructor) + ThisType & + Init(const size_t r, + const size_t c, bool clear=true); + + /** + * @brief Dealocates the matrix from memory and resets the dimensions to (0, 0) + */ + void + Destroy(); + + + ThisType & + Zero(); + + ThisType & + Unit(); // set to unit. + + /** + * @brief Copies the contents of a matrix + * @param rM Source data matrix + * @return Returns reference to this + */ + template<typename _ElemU> ThisType & + Copy(const Matrix<_ElemU> & rM, MatrixTrasposeType Trans=NO_TRANS); + + + + /** + * @brief Copies the elements of a vector row-by-row into a matrix + * @param rV Source vector + * @param nRows Number of rows of returned matrix + * @param nCols Number of columns of returned matrix + * + * Note that rV.Dim() must equal nRows*nCols + */ + ThisType & + CopyVectorSplicedRows(const Vector<_ElemT> &rV, const size_t nRows, const size_t nCols); + + /** + * @brief Returns @c true if matrix is initialized + */ + bool + IsInitialized() const + { return mpData != NULL; } + + /// Returns number of rows in the matrix + inline size_t + Rows() const + { + return mMRows; + } + + /// Returns number of columns in the matrix + inline size_t + Cols() const + { + return mMCols; + } + + /// Returns number of columns in the matrix memory + inline size_t + Stride() const + { + return mStride; + } + + + /** + * @brief Gives access to a specified matrix row without range check + * @return Pointer to the const array + */ + inline const _ElemT* __attribute__((aligned(16))) + pData () const + { + return mpData; + } + + + /** + * @brief Gives access to a specified matrix row without range check + * @return Pointer to the non-const data array + */ + inline _ElemT* __attribute__((aligned(16))) + pData () + { + return mpData; + } + + + /** + * @brief pData_workaround is a workaround that allows SubMatrix to get a + * @return pointer to non-const data even though the Matrix is const... + */ + protected: + inline _ElemT* __attribute__((aligned(16))) + pData_workaround () const + { + return mpData; + } + public: + + + /// Returns size of matrix in memory + size_t + MSize() const + { + return mMRows * mStride * sizeof(_ElemT); + } + + /// Checks the content of the matrix for nan and inf values + void + CheckData(const std::string file = "") const + { + for(size_t row=0; row<Rows(); row++) { + for(size_t col=0; col<Cols(); col++) { + if(isnan((*this)(row,col)) || isinf((*this)(row,col))) { + std::ostringstream os; + os << "Invalid value: " << (*this)(row,col) + << " in matrix row: " << row + << " col: " << col + << " file: " << file; + Error(os.str()); + } + } + } + } + + /** + * ********************************************************************** + * ********************************************************************** + * @defgroup RESHAPE Matrix reshaping rutines + * ********************************************************************** + * ********************************************************************** + * @{ + */ + + /** + * @brief Removes one row from the matrix. The memory is not reallocated. + */ + ThisType & + RemoveRow(size_t i); + + /** @} */ + + /** + * ********************************************************************** + * ********************************************************************** + * @defgroup ACCESS Access functions and operators + * ********************************************************************** + * ********************************************************************** + * @{ + */ + + /** + * @brief Gives access to a specified matrix row without range check + * @return Subvector object representing the row + */ + inline const SubVector<_ElemT> + operator [] (size_t i) const + { + assert(i < mMRows); + return SubVector<_ElemT>(mpData + (i * mStride), Cols()); + } + + inline SubVector<_ElemT> + operator [] (size_t i) + { + assert(i < mMRows); + return SubVector<_ElemT>(mpData + (i * mStride), Cols()); + } + + /** + * @brief Gives access to a specified matrix row without range check + * @return pointer to the first field of the row + */ + inline _ElemT* + pRowData(size_t i) + { + assert(i < mMRows); + return mpData + i * mStride; + } + + /** + * @brief Gives access to a specified matrix row without range check + * @return pointer to the first field of the row (const version) + */ + inline const _ElemT* + pRowData(size_t i) const + { + assert(i < mMRows); + return mpData + i * mStride; + } + + /** + * @brief Gives access to matrix elements (row, col) + * @return reference to the desired field + */ + inline _ElemT& + operator () (size_t r, size_t c) + { +#ifdef PARANOID + assert(r < mMRows && c < mMCols); +#endif + return *(mpData + r * mStride + c); + } + + /** + * @brief Gives access to matrix elements (row, col) + * @return pointer to the desired field (const version) + */ + inline const _ElemT + operator () (size_t r, size_t c) const + { +#ifdef PARANOID + assert(r < mMRows && c < mMCols); +#endif + return *(mpData + r * mStride + c); + } + + /** + * @brief Returns a matrix sub-range + * @param ro Row offset + * @param r Rows in range + * @param co Column offset + * @param c Coluns in range + * See @c SubMatrix class for details + */ + SubMatrix<_ElemT> + Range(const size_t ro, const size_t r, + const size_t co, const size_t c) + { return SubMatrix<_ElemT>(*this, ro, r, co, c); } + + const SubMatrix<_ElemT> + Range(const size_t ro, const size_t r, + const size_t co, const size_t c) const + { return SubMatrix<_ElemT>(*this, ro, r, co, c); } + /** @} */ + + + /** + * ********************************************************************** + * ********************************************************************** + * @defgroup MATH ROUTINES + * ********************************************************************** + * ********************************************************************** + * @{ + **/ + + /** + * @brief Returns sum of all elements + */ + _ElemT& + Sum() const; + + ThisType & + DotMul(const ThisType& a); + + ThisType & + Scale(_ElemT alpha); + + ThisType & + ScaleCols(const Vector<_ElemT> &scale); // Equivalent to (*this) = (*this) * diag(scale). + + ThisType & + ScaleRows(const Vector<_ElemT> &scale); // Equivalent to (*this) = diag(scale) * (*this); + + /// Sum another matrix rMatrix with this matrix + ThisType& + Add(const Matrix<_ElemT>& rMatrix); + + + /// Sum scaled matrix rMatrix with this matrix + ThisType& + AddScaled(_ElemT alpha, const Matrix<_ElemT>& rMatrix); + + /// Apply log to all items of the matrix + ThisType& + ApplyLog(); + + /** + * @brief Computes the determinant of this matrix + * @return Returns the determinant of a matrix + * @ingroup MATH + * + */ + _ElemT LogAbsDeterminant(_ElemT *DetSign=NULL); + + + /** + * @brief Performs matrix inplace inversion + */ + ThisType & + Invert(_ElemT *LogDet=NULL, _ElemT *DetSign=NULL, bool inverse_needed=true); + + /** + * @brief Performs matrix inplace inversion in double precision, even if this object is not double precision. + */ + ThisType & + InvertDouble(_ElemT *LogDet=NULL, _ElemT *DetSign=NULL, bool inverse_needed=true){ + double LogDet_tmp, DetSign_tmp; + Matrix<double> dmat(*this); dmat.Invert(&LogDet_tmp, &DetSign_tmp, inverse_needed); if(inverse_needed) (*this).Copy(dmat); + if(LogDet) *LogDet = LogDet_tmp; if(DetSign) *DetSign = DetSign_tmp; + return *this; + } + + + /** + * @brief Inplace matrix transposition. Applicable only to square matrices + */ + ThisType & + Transpose() + { + assert(Rows()==Cols()); + size_t M=Rows(); + for(size_t i=0;i<M;i++) + for(size_t j=0;j<i;j++){ + _ElemT &a = (*this)(i,j), &b = (*this)(j,i); + std::swap(a,b); + } + return *this; + } + + + + + + bool IsSymmetric(_ElemT cutoff = 1.0e-05) const; + + bool IsDiagonal(_ElemT cutoff = 1.0e-05) const; + + bool IsUnit(_ElemT cutoff = 1.0e-05) const; + + bool IsZero(_ElemT cutoff = 1.0e-05) const; + + _ElemT FrobeniusNorm() const; // sqrt of sum of square elements. + + _ElemT LargestAbsElem() const; // largest absolute value. + + + friend _ElemT TNet::TraceOfProduct<_ElemT>(const Matrix<_ElemT> &A, const Matrix<_ElemT> &B); // tr(A B) + friend _ElemT TNet::TraceOfProductT<_ElemT>(const Matrix<_ElemT> &A, const Matrix<_ElemT> &B); // tr(A B^T)==tr(A^T B) + friend class SubMatrix<_ElemT>; // so it can get around const restrictions on the pointer to mpData. + + /** ********************************************************************** + * ********************************************************************** + * @defgroup BLAS_ROUTINES BLAS ROUTINES + * @ingroup MATH + * ********************************************************************** + * ********************************************************************** + **/ + + ThisType & + BlasGer(const _ElemT alpha, const Vector<_ElemT>& rA, const Vector<_ElemT>& rB); + + ThisType & + Axpy(const _ElemT alpha, const Matrix<_ElemT> &rM, MatrixTrasposeType transA=NO_TRANS); + + ThisType & + BlasGemm(const _ElemT alpha, + const ThisType& rA, MatrixTrasposeType transA, + const ThisType& rB, MatrixTrasposeType transB, + const _ElemT beta = 0.0); + + + /** @} */ + + + /** ********************************************************************** + * ********************************************************************** + * @defgroup IO Input/Output ROUTINES + * ********************************************************************** + * ********************************************************************** + * @{ + **/ + + friend std::ostream & + operator << <> (std::ostream & out, const ThisType & m); + + void PrintOut(char *file); + void ReadIn(char *file); + + + bool + LoadHTK(const char* pFileName); + + /** @} */ + + + protected: +// inline void swap4b(void *a); +// inline void swap2b(void *a); + + + protected: + /// data memory area + _ElemT* mpData; + + /// these atributes store the real matrix size as it is stored in memory + /// including memalignment + size_t mMCols; ///< Number of columns + size_t mMRows; ///< Number of rows + size_t mStride; ///< true number of columns for the internal matrix. + ///< This number may differ from M_cols as memory + ///< alignment might be used + +#ifdef STK_MEMALIGN_MANUAL + /// data to be freed (in case of manual memalignment use, see Common.h) + _ElemT* mpFreeData; +#endif + }; // class Matrix + + template<> Matrix<float> & Matrix<float>::Invert(float *LogDet, float *DetSign, bool inverse_needed); // state that we will implement separately for float and double. + template<> Matrix<double> & Matrix<double>::Invert(double *LogDet, double *DetSign, bool inverse_needed); + + + + /** ************************************************************************** + ** ************************************************************************** + * @brief Sub-matrix representation + * + * This class provides a way to work with matrix cutouts in STK. + * + * + */ + template<typename _ElemT> + class SubMatrix : public Matrix<_ElemT> + { + typedef SubMatrix<_ElemT> ThisType; + + public: + /// Constructor + SubMatrix(const Matrix<_ElemT>& rT, // Input matrix cannot be const because SubMatrix can change its contents. + const size_t ro, + const size_t r, + const size_t co, + const size_t c); + + + /// The destructor + ~SubMatrix<_ElemT>() + { +#ifndef STK_MEMALIGN_MANUAL + Matrix<_ElemT>::mpData = NULL; +#else + Matrix<_ElemT>::mpFreeData = NULL; +#endif + } + + /// Assign operator + ThisType& operator=(const ThisType& rSrc) + { + //std::cout << "[PERFORMing operator= SubMatrix&^2]" << std::flush; + this->mpData = rSrc.mpData; + this->mMCols = rSrc.mMCols; + this->mMRows = rSrc.mMRows; + this->mStride = rSrc.mStride; + this->mpFreeData = rSrc.mpFreeData; + return *this; + } + + + + /// Initializes matrix (if not done by constructor) + ThisType & + Init(const size_t r, + const size_t c, bool clear=true) + { Error("Submatrix cannot do Init"); return *this; } + + /** + * @brief Dealocates the matrix from memory and resets the dimensions to (0, 0) + */ + void + Destroy() + { Error("Submatrix cannot do Destroy"); } + + + + }; + + + + //Create useful shortcuts + typedef Matrix<BaseFloat> BfMatrix; + typedef SubMatrix<BaseFloat> BfSubMatrix; + + /** + * Function for summing matrices of different types + */ + template<typename _ElemT, typename _ElemU> + void Add(Matrix<_ElemT>& rDst, const Matrix<_ElemU>& rSrc) { + assert(rDst.Cols() == rSrc.Cols()); + assert(rDst.Rows() == rSrc.Rows()); + + for(size_t i=0; i<rDst.Rows(); i++) { + const _ElemU* p_src = rSrc.pRowData(i); + _ElemT* p_dst = rDst.pRowData(i); + for(size_t j=0; j<rDst.Cols(); j++) { + *p_dst++ += (_ElemT)*p_src++; + } + } + } + + /** + * Function for summing matrices of different types + */ + template<typename _ElemT, typename _ElemU> + void AddScaled(Matrix<_ElemT>& rDst, const Matrix<_ElemU>& rSrc, _ElemT scale) { + assert(rDst.Cols() == rSrc.Cols()); + assert(rDst.Rows() == rSrc.Rows()); + + Vector<_ElemT> tmp(rDst[0]); + + for(size_t i=0; i<rDst.Rows(); i++) { + tmp.Copy(rSrc[i]); + rDst[i].BlasAxpy(scale, tmp); + + /* + const _ElemU* p_src = rSrc.pRowData(i); + _ElemT* p_dst = rDst.pRowData(i); + for(size_t j=0; j<rDst.Cols(); j++) { + *p_dst++ += (_ElemT)(*p_src++) * scale; + } + */ + } + } + + + + + +} // namespace STK + + + +//***************************************************************************** +//***************************************************************************** +// we need to include the implementation +#include "Matrix.tcc" +//***************************************************************************** +//***************************************************************************** + + +/****************************************************************************** + ****************************************************************************** + * The following section contains specialized template definitions + * whose implementation is in Matrix.cc + */ + + +//#ifndef TNet_Matrix_h +#endif diff --git a/tnet_io/KaldiLib/Matrix.tcc b/tnet_io/KaldiLib/Matrix.tcc new file mode 100644 index 0000000..f6ffb8f --- /dev/null +++ b/tnet_io/KaldiLib/Matrix.tcc @@ -0,0 +1,796 @@ + +/** @file Matrix.tcc + * This is an internal header file, included by other library headers. + * You should not attempt to use it directly. + */ + + +#ifndef TNet_Matrix_tcc +#define TNet_Matrix_tcc + +//#pragma GCC system_header + +#include <cstdlib> +#include <cmath> +#include <cfloat> +#include <fstream> +#include <iomanip> +#include <typeinfo> +#include <algorithm> +#include <limits> +#include <vector> +#include "Common.h" + +#ifndef _XOPEN_SOURCE + #define _XOPEN_SOURCE 600 +#endif + + +#ifdef HAVE_ATLAS +extern "C"{ + #include <cblas.h> +} +#endif + + +#include "Common.h" +#include "Vector.h" +namespace TNet +{ + +//****************************************************************************** + template<typename _ElemT> + Matrix<_ElemT> & + Matrix<_ElemT>:: + Init(const size_t rows, + const size_t cols, + bool clear) + { + if(mpData != NULL) Destroy(); + if(rows*cols == 0){ + assert(rows==0 && cols==0); + mMRows=rows; + mMCols=cols; +#ifdef STK_MEMALIGN_MANUAL + mpFreeData=NULL; +#endif + mpData=NULL; + return *this; + } + // initialize some helping vars + size_t skip; + size_t real_cols; + size_t size; + void* data; // aligned memory block + void* free_data; // memory block to be really freed + + // compute the size of skip and real cols + skip = ((16 / sizeof(_ElemT)) - cols % (16 / sizeof(_ElemT))) % (16 / sizeof(_ElemT)); + real_cols = cols + skip; + size = rows * real_cols * sizeof(_ElemT); + + // allocate the memory and set the right dimensions and parameters + + if (NULL != (data = stk_memalign(16, size, &free_data))) + { + mpData = static_cast<_ElemT *> (data); +#ifdef STK_MEMALIGN_MANUAL + mpFreeData = static_cast<_ElemT *> (free_data); +#endif + mMRows = rows; + mMCols = cols; + mStride = real_cols; + } + else + { + throw std::bad_alloc(); + } + if(clear) Zero(); + return *this; + } // + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + template<typename _ElemU> + Matrix<_ElemT> & + Matrix<_ElemT>:: + Copy(const Matrix<_ElemU> & rM, MatrixTrasposeType Trans) + { + if(Trans==NO_TRANS){ + assert(mMRows == rM.Rows() && mMCols == rM.Cols()); + for(size_t i = 0; i < mMRows; i++) + (*this)[i].Copy(rM[i]); + return *this; + } else { + assert(mMCols == rM.Rows() && mMRows == rM.Cols()); + for(size_t i = 0; i < mMRows; i++) + for(size_t j = 0; j < mMCols; j++) + (*this)(i,j) = rM(j,i); + return *this; + } + } + + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT> & + Matrix<_ElemT>:: + CopyVectorSplicedRows(const Vector<_ElemT> &rV, const size_t nRows, const size_t nCols) { + assert(rV.Dim() == nRows*nCols); + mMRows = nRows; + mMCols = nCols; + Init(nRows,nCols,true); + for(size_t r=0; r<mMRows; r++) + for(size_t c=0; c<mMCols; c++) + (*this)(r,c) = rV(r*mMCols + c); + + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT> & + Matrix<_ElemT>:: + RemoveRow(size_t i) + { + assert(i < mMRows && "Access out of matrix"); + for(size_t j = i + 1; j < mMRows; j++) + (*this)[j - 1].Copy((*this)[j]); + mMRows--; + return *this; + } + + + //**************************************************************************** + //**************************************************************************** + // The destructor + template<typename _ElemT> + void + Matrix<_ElemT>:: + Destroy() + { + // we need to free the data block if it was defined +#ifndef STK_MEMALIGN_MANUAL + if (NULL != mpData) free(mpData); +#else + if (NULL != mpData) free(mpFreeData); + mpFreeData = NULL; +#endif + + mpData = NULL; + mMRows = mMCols = 0; + } + + //**************************************************************************** + //**************************************************************************** +// template<typename _ElemT> +// void +// Matrix<_ElemT>:: +// VectorizeRows(Vector<_ElemT> &rV) { +//#ifdef PARANIOD +// assert(rV.Dim() == mMRows*mMCols); +//#endif +// for(size_t r=0; r<mMRows; r++) { +// rV.Range((r-1)*mMCols, mMCols).Copy((*this)[r]); +// } +// } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + bool + Matrix<_ElemT>:: + LoadHTK(const char* pFileName) + { + HtkHeader htk_hdr; + + FILE *fp = fopen(pFileName, "rb"); + if(!fp) + { + return false; + } + + read(fileno(fp), &htk_hdr, sizeof(htk_hdr)); + + swap4(htk_hdr.mNSamples); + swap4(htk_hdr.mSamplePeriod); + swap2(htk_hdr.mSampleSize); + swap2(htk_hdr.mSampleKind); + + Init(htk_hdr.mNSamples, htk_hdr.mSampleSize / sizeof(float)); + + size_t i; + size_t j; + if (typeid(_ElemT) == typeid(float)) + { + for (i=0; i< Rows(); ++i) { + read(fileno(fp), (*this).pRowData(i), Cols() * sizeof(float)); + + for(j = 0; j < Cols(); j++) { + swap4(((*this)(i,j))); + } + } + } + else + { + float *pmem = new (std::nothrow) float[Cols()]; + if (!pmem) + { + fclose(fp); + return false; + } + + for(i = 0; i < Rows(); i++) { + read(fileno(fp), pmem, Cols() * sizeof(float)); + + for (j = 0; j < Cols(); ++j) { + swap4(pmem[j]); + (*this)(i,j) = static_cast<_ElemT>(pmem[j]); + } + } + delete [] pmem; + } + + fclose(fp); + + return true; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT> & + Matrix<_ElemT>:: + DotMul(const ThisType& a) + { + size_t i; + size_t j; + + for (i = 0; i < mMRows; ++i) { + for (j = 0; j < mMCols; ++j) { + (*this)(i,j) *= a(i,j); + } + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + _ElemT & + Matrix<_ElemT>:: + Sum() const + { + double sum = 0.0; + + for (size_t i = 0; i < Rows(); ++i) { + for (size_t j = 0; j < Cols(); ++j) { + sum += (*this)(i,j); + } + } + + return sum; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT>& + Matrix<_ElemT>:: + Scale(_ElemT alpha) + { +#if 0 + for (size_t i = 0; i < Rows(); ++i) + for (size_t j = 0; j < Cols(); ++j) + (*this)(i,j) *= alpha; +#else + for (size_t i = 0; i < Rows(); ++i) { + _ElemT* p_data = pRowData(i); + for (size_t j = 0; j < Cols(); ++j) { + *p_data++ *= alpha; + } + } +#endif + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT>& + Matrix<_ElemT>:: + ScaleRows(const Vector<_ElemT>& scale) // scales each row by scale[i]. + { + assert(scale.Dim() == Rows()); + size_t M = Rows(), N = Cols(); + + for (size_t i = 0; i < M; i++) { + _ElemT this_scale = scale(i); + for (size_t j = 0; j < N; j++) { + (*this)(i,j) *= this_scale; + } + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT>& + Matrix<_ElemT>:: + ScaleCols(const Vector<_ElemT>& scale) // scales each column by scale[i]. + { + assert(scale.Dim() == Cols()); + for (size_t i = 0; i < Rows(); i++) { + for (size_t j = 0; j < Cols(); j++) { + _ElemT this_scale = scale(j); + (*this)(i,j) *= this_scale; + } + } + return *this; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT>& + Matrix<_ElemT>:: + Add(const Matrix<_ElemT>& rMatrix) + { + assert(rMatrix.Cols() == Cols()); + assert(rMatrix.Rows() == Rows()); + +#if 0 + //this can be slow + for (size_t i = 0; i < Rows(); i++) { + for (size_t j = 0; j < Cols(); j++) { + (*this)(i,j) += rMatrix(i,j); + } + } +#else + //this will be faster (but less secure) + for(size_t i=0; i<Rows(); i++) { + const _ElemT* p_src = rMatrix.pRowData(i); + _ElemT* p_dst = pRowData(i); + for(size_t j=0; j<Cols(); j++) { + *p_dst++ += *p_src++; + } + } +#endif + return *this; + } + + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT>& + Matrix<_ElemT>:: + AddScaled(_ElemT alpha, const Matrix<_ElemT>& rMatrix) + { + assert(rMatrix.Cols() == Cols()); + assert(rMatrix.Rows() == Rows()); + +#if 0 + //this can be slow + for (size_t i = 0; i < Rows(); i++) { + for (size_t j = 0; j < Cols(); j++) { + (*this)(i,j) += rMatrix(i,j) * alpha; + } + } +#else + /* + //this will be faster (but less secure) + for(size_t i=0; i<Rows(); i++) { + const _ElemT* p_src = rMatrix.pRowData(i); + _ElemT* p_dst = pRowData(i); + for(size_t j=0; j<Cols(); j++) { + *p_dst++ += *p_src++ * alpha; + } + } + */ + + //let's use BLAS + for(size_t i=0; i<Rows(); i++) { + (*this)[i].BlasAxpy(alpha, rMatrix[i]); + } +#endif + return *this; + } + + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT>& + Matrix<_ElemT>:: + ApplyLog() + { + +#if 0 + //this can be slow + for (size_t i = 0; i < Rows(); i++) { + for (size_t j = 0; j < Cols(); j++) { + (*this)(i,j) = += _LOG((*this)(i,j)); + } + } +#else + //this will be faster (but less secure) + for(size_t i=0; i<Rows(); i++) { + _ElemT* p_data = pRowData(i); + for(size_t j=0; j<Cols(); j++) { + *p_data = _LOG(*p_data); + p_data++; + } + } +#endif + return *this; + } + + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT> & + Matrix<_ElemT>:: + Zero() + { + for(size_t row=0;row<mMRows;row++) + memset(mpData + row*mStride, 0, sizeof(_ElemT)*mMCols); + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Matrix<_ElemT> & + Matrix<_ElemT>:: + Unit() + { + for(size_t row=0;row<std::min(mMRows,mMCols);row++){ + memset(mpData + row*mStride, 0, sizeof(_ElemT)*mMCols); + (*this)(row,row) = 1.0; + } + return *this; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + void + Matrix<_ElemT>:: + PrintOut(char* file) + { + FILE* f = fopen(file, "w"); + unsigned i,j; + fprintf(f, "%dx%d\n", this->mMRows, this->mMCols); + + for(i=0; i<this->mMRows; i++) + { + _ElemT* row = (*this)[i]; + + for(j=0; j<this->mStride; j++){ + fprintf(f, "%20.17f ",row[j]); + } + fprintf(f, "\n"); + } + + fclose(f); + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + void + Matrix<_ElemT>:: + ReadIn(char* file) + { + FILE* f = fopen(file, "r"); + int i = 0; + int j = 0; + fscanf(f, "%dx%d\n", &i,&j); + fprintf(stderr, "%dx%d\n", i,j); + + for(i=0; i<this->mMRows; i++) + { + _ElemT* row = (*this)[i]; + + for(j=0; j<this->mStride; j++){ + fscanf(f, "%f ",&row[j]); + } + //fprintf(f, "\n"); + } + + fclose(f); + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + void Save (std::ostream &rOut, const Matrix<_ElemT> &rM) + { + for (size_t i = 0; i < rM.Rows(); i++) { + for (size_t j = 0; j < rM.Cols(); j++) { + rOut << rM(i,j) << ' '; + } + rOut << '\n'; + } + if(rOut.fail()) + throw std::runtime_error("Failed to write matrix to stream"); + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + std::ostream & + operator << (std::ostream & rOut, const Matrix<_ElemT> & rM) + { + rOut << "m " << rM.Rows() << ' ' << rM.Cols() << '\n'; + Save(rOut, rM); + return rOut; + } + + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + void Load (std::istream & rIn, Matrix<_ElemT> & rM) + { + if(MatrixVectorIostreamControl::Flags(rIn, ACCUMULATE_INPUT)) { + for (size_t i = 0; i < rM.Rows(); i++) { + std::streamoff pos = rIn.tellg(); + for (size_t j = 0; j < rM.Cols(); j++) { + _ElemT tmp; + rIn >> tmp; + rM(i,j) += tmp; + if(rIn.fail()){ + throw std::runtime_error("Failed to read matrix from stream. File position is "+to_string(pos)); + } + } + } + } else { + for (size_t i = 0; i < rM.Rows(); i++) { + std::streamoff pos = rIn.tellg(); + for (size_t j = 0; j < rM.Cols(); j++) { + rIn >> rM(i,j); + if(rIn.fail()){ + throw std::runtime_error("Failed to read matrix from stream. File position is "+to_string(pos)); + } + + } + } + } + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + std::istream & + operator >> (std::istream & rIn, Matrix<_ElemT> & rM) + { + while(isascii(rIn.peek()) && isspace(rIn.peek())) rIn.get(); // eat up space. + if(rIn.peek() == 'm'){ // "new" format: m <nrows> <ncols> \n 1.0 0.2 4.3 ... + rIn.get();// eat up the 'm'. + long long int nrows=-1; rIn>>nrows; + long long int ncols=-1; rIn>>ncols; + if(rIn.fail()||nrows<0||ncols<0){ throw std::runtime_error("Failed to read matrix from stream: no size\n"); } + + size_t nrows2 = size_t(nrows), ncols2 = size_t(ncols); + assert((long long int)nrows2 == nrows && (long long int)ncols2 == ncols); + + if(rM.Rows()!=nrows2 || rM.Cols()!=ncols2) rM.Init(nrows2,ncols2); + } + Load(rIn,rM); + return rIn; + } + + + + //**************************************************************************** + //**************************************************************************** + // Constructor + template<typename _ElemT> + SubMatrix<_ElemT>:: + SubMatrix(const Matrix<_ElemT>& rT, // Matrix cannot be const because SubMatrix can change its contents. Would have to have a ConstSubMatrix or something... + const size_t ro, + const size_t r, + const size_t co, + const size_t c) + { + assert(ro >= 0 && ro <= rT.Rows()); + assert(co >= 0 && co <= rT.Cols()); + assert(r > 0 && r <= rT.Rows() - ro); + assert(c > 0 && c <= rT.Cols() - co); + // point to the begining of window + Matrix<_ElemT>::mMRows = r; + Matrix<_ElemT>::mMCols = c; + Matrix<_ElemT>::mStride = rT.Stride(); + Matrix<_ElemT>::mpData = rT.pData_workaround() + co + ro * rT.Stride(); + } + + + +#ifdef HAVE_ATLAS + + template<> + Matrix<float> & + Matrix<float>:: + BlasGer(const float alpha, const Vector<float>& rA, const Vector<float>& rB); + + + template<> + Matrix<double> & + Matrix<double>:: + BlasGer(const double alpha, const Vector<double>& rA, const Vector<double>& rB); + + + template<> + Matrix<float>& + Matrix<float>:: + BlasGemm(const float alpha, + const Matrix<float>& rA, MatrixTrasposeType transA, + const Matrix<float>& rB, MatrixTrasposeType transB, + const float beta); + + template<> + Matrix<double>& + Matrix<double>:: + BlasGemm(const double alpha, + const Matrix<double>& rA, MatrixTrasposeType transA, + const Matrix<double>& rB, MatrixTrasposeType transB, + const double beta); + + template<> + Matrix<float>& + Matrix<float>:: + Axpy(const float alpha, + const Matrix<float>& rA, MatrixTrasposeType transA); + + template<> + Matrix<double>& + Matrix<double>:: + Axpy(const double alpha, + const Matrix<double>& rA, MatrixTrasposeType transA); + + template <> // non-member so automatic namespace lookup can occur. + double TraceOfProduct(const Matrix<double> &A, const Matrix<double> &B); + + template <> // non-member so automatic namespace lookup can occur. + double TraceOfProductT(const Matrix<double> &A, const Matrix<double> &B); + + template <> // non-member so automatic namespace lookup can occur. + float TraceOfProduct(const Matrix<float> &A, const Matrix<float> &B); + + template <> // non-member so automatic namespace lookup can occur. + float TraceOfProductT(const Matrix<float> &A, const Matrix<float> &B); + + + +#else // HAVE_ATLAS + #error Routines in this section are not implemented yet without BLAS +#endif // HAVE_ATLAS + + template<class _ElemT> + bool + Matrix<_ElemT>:: + IsSymmetric(_ElemT cutoff) const { + size_t R=Rows(), C=Cols(); + if(R!=C) return false; + _ElemT bad_sum=0.0, good_sum=0.0; + for(size_t i=0;i<R;i++){ + for(size_t j=0;j<i;j++){ + _ElemT a=(*this)(i,j),b=(*this)(j,i), avg=0.5*(a+b), diff=0.5*(a-b); + good_sum += fabs(avg); bad_sum += fabs(diff); + } + good_sum += fabs((*this)(i,i)); + } + if(bad_sum > cutoff*good_sum) return false; + return true; + } + + template<class _ElemT> + bool + Matrix<_ElemT>:: + IsDiagonal(_ElemT cutoff) const{ + size_t R=Rows(), C=Cols(); + _ElemT bad_sum=0.0, good_sum=0.0; + for(size_t i=0;i<R;i++){ + for(size_t j=0;j<C;j++){ + if(i==j) good_sum += (*this)(i,j); + else bad_sum += (*this)(i,j); + } + } + return (!(bad_sum > good_sum * cutoff)); + } + + template<class _ElemT> + bool + Matrix<_ElemT>:: + IsUnit(_ElemT cutoff) const { + size_t R=Rows(), C=Cols(); + if(R!=C) return false; + _ElemT bad_sum=0.0; + for(size_t i=0;i<R;i++) + for(size_t j=0;j<C;j++) + bad_sum += fabs( (*this)(i,j) - (i==j?1.0:0.0)); + return (bad_sum <= cutoff); + } + + template<class _ElemT> + bool + Matrix<_ElemT>:: + IsZero(_ElemT cutoff)const { + size_t R=Rows(), C=Cols(); + _ElemT bad_sum=0.0; + for(size_t i=0;i<R;i++) + for(size_t j=0;j<C;j++) + bad_sum += fabs( (*this)(i,j) ); + return (bad_sum <= cutoff); + } + + template<class _ElemT> + _ElemT + Matrix<_ElemT>:: + FrobeniusNorm() const{ + size_t R=Rows(), C=Cols(); + _ElemT sum=0.0; + for(size_t i=0;i<R;i++) + for(size_t j=0;j<C;j++){ + _ElemT tmp = (*this)(i,j); + sum += tmp*tmp; + } + return sqrt(sum); + } + + template<class _ElemT> + _ElemT + Matrix<_ElemT>:: + LargestAbsElem() const{ + size_t R=Rows(), C=Cols(); + _ElemT largest=0.0; + for(size_t i=0;i<R;i++) + for(size_t j=0;j<C;j++) + largest = std::max(largest, (_ElemT)fabs((*this)(i,j))); + return largest; + } + + + + // Uses SVD to compute the eigenvalue decomposition of a symmetric positive semidefinite + // matrix: + // (*this) = rU * diag(rS) * rU^T, with rU an orthogonal matrix so rU^{-1} = rU^T. + // Does this by computing svd (*this) = U diag(rS) V^T ... answer is just U diag(rS) U^T. + // Throws exception if this failed to within supplied precision (typically because *this was not + // symmetric positive definite). + + + + template<class _ElemT> + _ElemT + Matrix<_ElemT>:: + LogAbsDeterminant(_ElemT *DetSign){ + _ElemT LogDet; + Matrix<_ElemT> tmp(*this); + tmp.Invert(&LogDet, DetSign, false); // false== output not needed (saves some computation). + return LogDet; + } + +}// namespace TNet + +// #define TNet_Matrix_tcc +#endif diff --git a/tnet_io/KaldiLib/MlfStream.cc b/tnet_io/KaldiLib/MlfStream.cc new file mode 100644 index 0000000..a2f6478 --- /dev/null +++ b/tnet_io/KaldiLib/MlfStream.cc @@ -0,0 +1,268 @@ +#include "MlfStream.h" +#include "Common.h" +#include "Error.h" + + +namespace TNet +{ + //****************************************************************************** + LabelContainer:: + ~LabelContainer() + { + while (!this->mLabelList.empty()) + { + delete this->mLabelList.back(); + this->mLabelList.pop_back(); + } + } + + //****************************************************************************** + size_t + LabelContainer:: + DirDepth(const std::string & rPath) + { + size_t depth = 0; + size_t length = rPath.length(); + const char * s = rPath.c_str(); + + for (size_t i = 0; i < length; i++) + { + if (*s == '/' || *s == '\\') + { + depth++; + } + s++; + } + return depth; + } + + + //****************************************************************************** + void + LabelContainer:: + Insert(const std::string & rLabel, + std::streampos Pos) + { + LabelRecord ls; + size_t depth; + LabelRecord tmp_ls; + + // we need to compute the depth of the label path if + // wildcard is used + // do we have a wildcard??? + if (rLabel[0] == '*') + { + depth = this->DirDepth(rLabel); + } + else + { + depth = MAX_LABEL_DEPTH; + } + + // perhaps we want to store the depth of the path in the label for the wildcards + // to work + this->mDepths.insert(depth); + + // store the values + ls.mStreamPos = Pos; + ls.miLabelListLimit = mLabelList.end(); + + + if (mLabelList.begin() != mLabelList.end()) { + ls.miLabelListLimit--; + } + + // if no wildcard chars, then we try to store in hash, otherwise store in + // list + if (rLabel.find_first_of("*?%",1) == rLabel.npos) + { + if (!Find(rLabel, tmp_ls)) + { + // look in the + this->mLabelMap[rLabel] = ls; + } + else { + ; + //Warning("More general definition found when inserting " + rLabel + " ... label: " + MatchedPattern()); + } + } + else + { + this->mLabelList.push_back(new std::pair<std::string,LabelRecord>(rLabel, ls)); + } + } + + + //****************************************************************************** + bool + LabelContainer:: + FindInHash(const std::string & rLabel, LabelRecord & rLS) + { + bool found = false; + + std::string str; + + // current depth within the str + DepthType current_depth = MAX_LABEL_DEPTH; + + // current search position within the str + size_t prev = rLabel.size() + 1; + + // we will walk through the set depts bacwards so we begin at the end and move + // to the front... + std::set<DepthType>::reverse_iterator ri (this->mDepths.end()); + std::set<DepthType>::reverse_iterator rlast (this->mDepths.begin()); + LabelHashType::iterator lab; + + // we perform the search until we run to the end of the set or we find something + while ((!found) && (ri != rlast)) + { + // we don't need to do anything with the string if the depth is set to + // max label depth since it contains no * + if (*ri == MAX_LABEL_DEPTH) + { + found = ((lab=this->mLabelMap.find(rLabel)) != this->mLabelMap.end()); + if (found) str = rLabel; + } + // we will crop the string and put * in the begining and try to search + else + { + // we know that we walk backwards in the depths, so we need to first find + // the last / and + if (current_depth == MAX_LABEL_DEPTH) + { + if (*ri > 0) + { + // we find the ri-th / from back + for (DepthType i=1; (i <= *ri) && (prev != rLabel.npos); i++) + { + prev = rLabel.find_last_of("/\\", prev-1); + } + } + else + { + prev = 0; + } + + // check if finding succeeded (prev == str.npos => failure, see STL) + if (prev != rLabel.npos) + { + // construct the new string beign sought for + str.assign(rLabel, prev, rLabel.size()); + str = '*' + str; + + // now we try to find + found = ((lab=this->mLabelMap.find(str)) != this->mLabelMap.end()); + + // say, that current depth is *ri + current_depth = *ri; + } + else + { + prev = rLabel.size() + 1; + } + } // if (current_depth == MAX_LABEL_DEPTH) + else + { + // now we know at which / we are from the back, so we search forward now + // and we need to reach the ri-th / + while (current_depth > *ri) + { + // we try to find next / + if ((prev = rLabel.find_first_of("/\\", prev+1)) != rLabel.npos) + current_depth--; + else + return false; + } + + // construct the new string beign sought for + str.assign(rLabel, prev, rLabel.size()); + str = '*' + str; + + // now we try to find + found = ((lab=this->mLabelMap.find(str)) != this->mLabelMap.end()); + } + } + + // move one element further (jump to next observed depth) + ri++; + } // while (run) + + // some debug info + if (found) + { + rLS = lab->second; + this->mMatchedPattern = str; + } + + return found; + } + + + //****************************************************************************** + bool + LabelContainer:: + FindInList(const std::string & rLabel, LabelRecord & rLS, bool limitSearch) + { + + bool found = false; + std::string str; + LabelListType::iterator lab = mLabelList.begin(); + LabelListType::iterator limit; + + if (limitSearch && (rLS.miLabelListLimit != mLabelList.end())) + { + limit = rLS.miLabelListLimit; + limit++; + } + else + { + limit = this->mLabelList.end(); + } + + // we perform sequential search until we run to the end of the list or we find + // something + while ((!found) && (lab != limit)) + { + if (ProcessMask(rLabel, (*lab)->first, str)) + { + found = true; + } + else + { + lab++; + } + } // while (run) + + // some debug info + if (found) + { + rLS = (*lab)->second; + this->mMatchedPattern = (*lab)->first; + this->mMatchedPatternMask = str; + } + return found; + } + + + //****************************************************************************** + bool + LabelContainer:: + Find(const std::string & rLabel, LabelRecord & rLS) + { + // try to find the label in the Hash + if (FindInHash(rLabel, rLS)) + { + // we look in the list, but we limit the search. + FindInList(rLabel, rLS, true); + return true; + } //if (this->mLabelContainer.FindInHash(rLabel, label_stream)) + else + { + // we didn't find it in the hash so we look in the list + return FindInList(rLabel, rLS); + } + } + +} // namespace TNet + diff --git a/tnet_io/KaldiLib/MlfStream.h b/tnet_io/KaldiLib/MlfStream.h new file mode 100644 index 0000000..d643f5c --- /dev/null +++ b/tnet_io/KaldiLib/MlfStream.h @@ -0,0 +1,639 @@ +/** @file MlfStream.h + * This is an TNet C++ Library header. + * + * The naming convention in this file coppies the std::* naming as well as STK + */ + + +#ifndef STK_MlfStream_h +#define STK_MlfStream_h + +#include <iostream> +#include <vector> +#include <map> +#include <list> +#include <set> + + +namespace TNet +{ + class LabelRecord; + class LabelContainer; + + + /// this container stores the lables in linear order as they came + /// i.e. they cannot be hashed + typedef std::list< std::pair<std::string,LabelRecord> *> LabelListType; + + /// type of the container used to store the labels + typedef std::map<std::string, LabelRecord> LabelHashType; + + + + /** + * @brief Describes type of MLF definition + * + * See HTK book for MLF structure. Terms used in TNet are + * compatible with those in HTK book. + */ + enum MlfDefType + { + MLF_DEF_UNKNOWN = 0, ///< unknown definition + MLF_DEF_IMMEDIATE_TRANSCRIPTION, ///< immediate transcription + MLF_DEF_SUB_DIR_DEF ///< subdirectory definition + }; + + + + /** ************************************************************************** + * @brief Holds association between label and stream + */ + class LabelRecord + { + + public: + LabelRecord() : miLabelListLimit(NULL) + { } + + ~LabelRecord() + { } + + /// definition type + MlfDefType mDefType; + + /// position of the label in the stream + std::streampos mStreamPos; + + /** + * @brief points to the current end of the LabelList + * + * The reason for storing this value is to know when we inserted + * a label into the hash. It is possible, that the hash label came + * after list label, in which case the list label is prefered + */ + LabelListType::iterator miLabelListLimit; + + }; + + + + + /** + * @brief Provides an interface to label hierarchy and searching + * + * This class stores label files in a map structure. When a wildcard + * convence is used, the class stores the labels in separate maps according + * to level of wildcard abstraction. By level we mean the directory structure + * depth. + */ + class LabelContainer + { + public: + /// The constructor + LabelContainer() : mUseHashedSearch(true) {} + + /// The destructor + ~LabelContainer(); + + /** + * @brief Inserts new label to the hash structure + */ + void + Insert( + const std::string & rLabel, + std::streampos Pos); + + + /** + * @brief Looks for a record in the hash + */ + bool + FindInHash( + const std::string& rLabel, + LabelRecord& rLS); + + /** + * @brief Looks for a record in the list + * @param rLabel Label to look for + * @param rLS Structure to fill with found data + * @param limitSearch If true @p rLS's @c mLabelListLimit gives the limiting position in the list + */ + bool + FindInList( + const std::string& rLabel, + LabelRecord& rLS, + bool limitSearch = false); + + /** + * @brief Looks for a record + */ + bool + Find( + const std::string & rLabel, + LabelRecord & rLS); + + /** + * @brief Returns the matched pattern + */ + const std::string & + MatchedPattern() const + { + return mMatchedPattern; + } + + /** + * @brief Returns the matched pattern mask (%%%) + */ + const std::string & + MatchedPatternMask() const + { + return mMatchedPatternMask; + } + + /** + * @brief Writes contents to stream (text) + * @param rOStream stream to write to + */ + void + Write(std::ostream& rOStream); + + private: + /// type used for directory depth notation + typedef size_t DepthType; + + + /// this set stores depths of * labels observed at insertion + std::set<DepthType> mDepths; + + /// stores the labels + LabelHashType mLabelMap; + LabelListType mLabelList; + + /// true if labels are to be sought by hashing function (fast) or by + /// sequential search (slow) + bool mUseHashedSearch; + + /// if Find matches the label, this var stores the pattern that matched the + /// query + std::string mMatchedPattern; + + /// if Find matches the label, this var stores the the masked characters. + /// The mask is given by '%' symbols + std::string mMatchedPatternMask; + + /** + * @brief Returns the directory depth of path + */ + size_t + DirDepth(const std::string & path); + + + }; + + + /** + * @brief MLF output buffer definition + */ + template< + typename _CharT, + typename _Traits = std::char_traits<_CharT>, + typename _CharTA = std::allocator<_CharT>, + typename ByteT = char, + typename ByteAT = std::allocator<ByteT> + > + class BasicOMlfStreamBuf + : public std::basic_streambuf<_CharT, _Traits> + { + public: + // necessary typedefs .................................................... + typedef BasicOMlfStreamBuf<_CharT,_Traits,_CharTA,ByteT,ByteAT> + this_type; + typedef std::basic_ostream<_CharT, _Traits>& + OStreamReference; + typedef std::basic_streambuf<_CharT, _Traits> + StreamBufType; + typedef _CharTA char_allocator_type; + typedef _CharT char_type; + typedef typename _Traits::int_type int_type; + typedef typename _Traits::pos_type pos_type; + typedef ByteT byte_type; + typedef ByteAT byte_allocator_type; + typedef byte_type* byte_buffer_type; + typedef std::vector<byte_type, byte_allocator_type > byte_vector_type; + typedef std::vector<char_type, char_allocator_type > char_vector_type; + + + BasicOMlfStreamBuf(OStreamReference rOStream, size_t bufferSize); + + ~BasicOMlfStreamBuf(); + + // virtual functions inherited from basic_streambuf....................... + int + sync(); + + /** + * @brief Write character in the case of overflow + * @param c Character to be written. + * @return A value different than EOF (or traits::eof() for other traits) + * signals success. If the function fails, either EOF + * (or traits::eof() for other traits) is returned or an + * exception is thrown. + */ + int_type + overflow(int_type c = _Traits::eof()); + + + // MLF specific functions ................................................ + /** + * @brief Creates a new MLF block + * @param rFileName filename to be opened + */ + this_type* + Open(const std::string& rFileName); + + /** + * @brief Closes MLF block + */ + void + Close(); + + /** + * @brief Returns true if the MLF is now in open state + */ + bool + IsOpen() const + { return mIsOpen; } + + LabelContainer& + rLabels() + { return mLabels; } + + private: + bool mIsOpen; + char_type mLastChar; + OStreamReference mOStream; + LabelContainer mLabels; + }; // class BasicOMlfStreamBuf + + + + /** + * @brief MLF input buffer definition + */ + template< + typename _CharT, + typename _Traits = std::char_traits<_CharT>, + typename _CharTA = std::allocator<_CharT>, + typename ByteT = char, + typename ByteAT = std::allocator<ByteT> + > + class BasicIMlfStreamBuf + : public std::basic_streambuf<_CharT, _Traits> + { + private: + // internal automaton states + static const int IN_HEADER_STATE = 0; + static const int OUT_OF_BODY_STATE = 1; + static const int IN_TITLE_STATE = 2; + static const int IN_BODY_STATE = 3; + + + public: // necessary typedefs .............................................. + typedef BasicIMlfStreamBuf<_CharT,_Traits,_CharTA,ByteT,ByteAT> + this_type; + typedef std::basic_istream<_CharT, _Traits>& IStreamReference; + typedef std::basic_streambuf<_CharT, _Traits> + StreamBufType; + typedef _CharTA char_allocator_type; + typedef _CharT char_type; + typedef typename _Traits::int_type int_type; + typedef typename _Traits::pos_type pos_type; + typedef ByteT byte_type; + typedef ByteAT byte_allocator_type; + typedef byte_type* byte_buffer_type; + typedef std::vector<byte_type, byte_allocator_type > byte_vector_type; + typedef std::vector<char_type, char_allocator_type > char_vector_type; + + + public: + // constructors and destructors .......................................... + BasicIMlfStreamBuf(IStreamReference rIStream, size_t bufferSize = 1024); + + ~BasicIMlfStreamBuf(); + + // virtual functions inherited from basic_streambuf....................... + /** + * @brief Get character in the case of underflow + * + * @return The new character available at the get pointer position, if + * any. Otherwise, traits::eof() is returned. + */ + int_type + underflow(); + + + // MLF specific functions ................................................ + /** + * @brief Creates a new MLF block + * @param rFileName filename to be opened + */ + this_type* + Open(const std::string& rFileName); + + /** + * @brief Closes MLF block + */ + this_type* + Close(); + + /** + * @brief Returns true if the MLF is now in open state + */ + bool + IsOpen() const + { return mIsOpen; } + + /** + * @brief Parses the stream (if possible) and stores positions to the + * label titles + */ + void + Index(); + + bool + IsHashed() const + { return mIsHashed; } + + /** + * @brief Jumps to next label definition + * @param rName std::string to be filled with the label name + * @return true on success + * + * The procedure automatically tries to hash the labels. + */ + bool + JumpToNextDefinition(std::string& rName); + + /** + * @brief Returns reference to the base stream + * @return reference to the stream + * + */ + IStreamReference + GetBaseStream() + { + return mIStream; + } + + private: // auxillary functions ............................................ + /** + * @brief Fills the line buffer with next line and updates the internal + * state of the finite automaton + */ + void + FillLineBuffer(); + + + private: // atributes ...................................................... + // some flags + bool mIsOpen; + bool mIsHashed; + bool mIsEof; + + /// internal state of the finite automaton + int mState; + + IStreamReference mIStream; + LabelContainer mLabels; + + std::vector<char_type> mLineBuffer; + }; // class BasicIMlfStreamBuf + + + + + /** + * @brief Base class with type-independent members for the Mlf Output + * Stram class + * + * This is a derivative of the basic_ios class. We derive it as we need + * to override some member functions + */ + template< + typename Elem, + typename Tr = std::char_traits<Elem>, + typename ElemA = std::allocator<Elem>, + typename ByteT = char, + typename ByteAT = std::allocator<ByteT> + > + class BasicOMlfStreamBase + : virtual public std::basic_ios<Elem,Tr> + { + public: + typedef std::basic_ostream<Elem, Tr>& OStreamReference; + typedef BasicOMlfStreamBuf < + Elem,Tr,ElemA,ByteT,ByteAT> OMlfStreamBufType; + + /** + * @brief constructor + * + * @param rOStream user defined output stream + */ + BasicOMlfStreamBase(OStreamReference rOStream, + size_t bufferSize) + : mBuf(rOStream, bufferSize) + { this->init(&mBuf); }; + + /** + * @brief Returns a pointer to the buffer object for this stream + */ + OMlfStreamBufType* + rdbuf() + { return &mBuf; }; + + private: + OMlfStreamBufType mBuf; + }; + + + template< + typename Elem, + typename Tr = std::char_traits<Elem>, + typename ElemA = std::allocator<Elem>, + typename ByteT = char, + typename ByteAT = std::allocator<ByteT> + > + class BasicIMlfStreamBase + : virtual public std::basic_ios<Elem,Tr> + { + public: + typedef std::basic_istream<Elem, Tr>& IStreamReference; + typedef BasicIMlfStreamBuf < + Elem,Tr,ElemA,ByteT,ByteAT> IMlfStreamBufType; + + BasicIMlfStreamBase( IStreamReference rIStream, + size_t bufferSize) + : mBuf(rIStream, bufferSize) + { this->init(&mBuf ); }; + + IMlfStreamBufType* + rdbuf() + { return &mBuf; }; + + IStreamReference + GetBaseStream() + { return mBuf.GetBaseStream(); } + + private: + IMlfStreamBufType mBuf; + }; + + + template< + typename Elem, + typename Tr = std::char_traits<Elem>, + typename ElemA = std::allocator<Elem>, + typename ByteT = char, + typename ByteAT = std::allocator<ByteT> + > + class BasicOMlfStream + : public BasicOMlfStreamBase<Elem,Tr,ElemA,ByteT,ByteAT>, + public std::basic_ostream<Elem,Tr> + { + public: + typedef BasicOMlfStreamBase< Elem,Tr,ElemA,ByteT,ByteAT> + BasicOMlfStreamBaseType; + typedef std::basic_ostream<Elem,Tr> OStreamType; + typedef OStreamType& OStreamReference; + + BasicOMlfStream(OStreamReference rOStream, size_t bufferSize = 32) + : BasicOMlfStreamBaseType(rOStream, bufferSize), + OStreamType(BasicOMlfStreamBaseType::rdbuf()) + { } + + /** + * @brief Destructor closes the stream + */ + ~BasicOMlfStream() + { } + + + /** + * @brief Creates a new MLF block + * @param rFileName filename to be opened + */ + void + Open(const std::string& rFileName) + { BasicOMlfStreamBaseType::rdbuf()->Open(rFileName); } + + /** + * @brief Closes MLF block + */ + void + Close() + { BasicOMlfStreamBaseType::rdbuf()->Close(); } + + /** + * @brief Returns true if the MLF is now in open state + */ + bool + IsOpen() const + { return BasicOMlfStreamBaseType::rdbuf()->IsOpen(); } + + /** + * @brief Accessor to the label container + * @return Reference to the label container + */ + LabelContainer& + rLabels() + { return BasicOMlfStreamBaseType::rdbuf()->rLabels(); } + }; + + + + template< + typename Elem, + typename Tr = std::char_traits<Elem>, + typename ElemA = std::allocator<Elem>, + typename ByteT = char, + typename ByteAT = std::allocator<ByteT> + > + class BasicIMlfStream + : public BasicIMlfStreamBase<Elem,Tr,ElemA,ByteT,ByteAT>, + public std::basic_istream<Elem,Tr> + { + public: + typedef BasicIMlfStreamBase <Elem,Tr,ElemA,ByteT,ByteAT> + BasicIMlfStreamBaseType; + typedef std::basic_istream<Elem,Tr> IStreamType; + typedef IStreamType& IStreamReference; + typedef unsigned char byte_type; + + BasicIMlfStream(IStreamReference rIStream, size_t bufferSize = 32) + : BasicIMlfStreamBaseType(rIStream, bufferSize), + IStreamType(BasicIMlfStreamBaseType::rdbuf()) + {}; + + + /** + * @brief Creates a new MLF block + * @param rFileName filename to be opened + */ + void + Open(const std::string& rFileName) + { + std::basic_streambuf<Elem, Tr>* p_buf; + + p_buf = BasicIMlfStreamBaseType::rdbuf()->Open(rFileName); + + if (NULL == p_buf) { + IStreamType::clear(IStreamType::rdstate() | std::ios::failbit); + } + else { + IStreamType::clear(); + } + } + + /** + * @brief Closes MLF block. + * In fact, nothing is done + */ + void + Close() + { + if (NULL == BasicIMlfStreamBaseType::rdbuf()->Close()) { + IStreamType::clear(IStreamType::rdstate() | std::ios::failbit); + } + } + + void + Index() + { BasicIMlfStreamBaseType::rdbuf()->Index(); } + + bool + IsHashed() const + { return BasicIMlfStreamBaseType::rdbuf()->IsHashed(); } + + }; + + + + // MAIN TYPEDEFS.............................................................. + typedef BasicOMlfStream<char> OMlfStream; + typedef BasicOMlfStream<wchar_t> WOMlfStream; + typedef BasicIMlfStream<char> IMlfStream; + typedef BasicIMlfStream<wchar_t> WIMlfStream; + + +#ifdef PATH_MAX + const size_t MAX_LABEL_DEPTH = PATH_MAX; +#else + const size_t MAX_LABEL_DEPTH = 1024; +#endif + + +} // namespace TNet + +#include "MlfStream.tcc" + +#endif diff --git a/tnet_io/KaldiLib/MlfStream.tcc b/tnet_io/KaldiLib/MlfStream.tcc new file mode 100644 index 0000000..8978545 --- /dev/null +++ b/tnet_io/KaldiLib/MlfStream.tcc @@ -0,0 +1,517 @@ +#ifndef STK_MlfStream_tcc +#define STK_MlfStream_tcc + +#include <algorithm> + +#include "Common.h" +#include "StkMatch.h" + +namespace TNet +{ + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + BasicOMlfStreamBuf(OStreamReference rOStream, size_t bufferSize) + : mIsOpen(false), mOStream(rOStream) + { } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + ~BasicOMlfStreamBuf() + { + mOStream.flush(); + } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + int + BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + sync() + { + mOStream.flush(); + return 0; + } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + typename _Traits::int_type + BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + overflow(typename _Traits::int_type c) + { + // we don't use buffer here... + if (mIsOpen) { + if (_Traits::eof() == c) { + return _Traits::not_eof(c); + } + // only pass the character to the stream + mOStream.rdbuf()->sputc(c); + + // remember last char (in case we want to close) + mLastChar = c; + + return c; + } + else { + return _Traits::eof(); + } + } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + void + BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + Close() + { + // if last character was not EOL, we need to insert it + if (mLastChar != '\n') { + mOStream.put('\n'); + } + mOStream << ".\n"; + + // flush the stream and declare the stream closed + mOStream.flush(); + mIsOpen = false; + } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT> * + BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + Open(const std::string& rFileName) + { + // retreive position + std::streampos pos = mOStream.tellp(); + + // write the initial "filename" in parantheses + mOStream << '"' << rFileName << '"' << std::endl; + mLastChar = '\n'; + + // return NULL if we canot open + if (!mOStream.good()) { + return NULL; + } + + // if ok, store the name position + if (-1 != pos) { + pos = mOStream.tellp(); + mLabels.Insert(rFileName, pos); + } + + // set open flag and return this + mIsOpen = true; + return this; + } + + + //**************************************************************************** + //**************************************************************************** + // BasicIMlfStreamBuf section + // + //**************************************************************************** + //**************************************************************************** + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + BasicIMlfStreamBuf(IStreamReference rIStream, size_t bufferSize) + : mIsOpen(false), mIsHashed(false), mIsEof(true), mState(IN_HEADER_STATE), + mIStream(rIStream), mLineBuffer() + { + // we reserve some place for the buffer... + mLineBuffer.reserve(bufferSize); + + //StreamBufType::setg(mpBuffer, mpBuffer + bufferSize, mpBuffer + bufferSize); + StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.back()), &(mLineBuffer.back())); + } + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + ~BasicIMlfStreamBuf() + { + } + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + void + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + Index() + { + // retreive position + std::streampos orig_pos = mIStream.tellg(); + int orig_state = mState; + + // for streams like stdin, pos will by definition be -1, so we can only + // rely on sequential access and cannot hash it. + if (-1 != orig_pos) { + std::string aux_name; + // we will constantly jump to next definition. the function automatically + // hashes the stream if possible + while (JumpToNextDefinition(aux_name)) + { } + + // move to the original position + mIStream.clear(); + mIStream.seekg(orig_pos); + mState = orig_state; + + // set as hashed + mIsHashed=true; + } + } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + bool + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + JumpToNextDefinition(std::string& rName) + { + if (!mIStream.good()) { + return false; + } + + // if we can, we will try to index the label + std::streampos pos = mIStream.tellg(); + + // we might be at a definition already, so first move one line further + FillLineBuffer(); + + // read lines till we get to definition again + while (mIStream.good() && mState != IN_TITLE_STATE) { + FillLineBuffer(); + } + + // decide what happened + if (IN_TITLE_STATE == mState) { + // if we can, we will try to index the label + pos = mIStream.tellg(); + + if (pos != static_cast<const std::streampos>(-1)) { + // if (pos !=std::string::npos) { // This line does not work under MinGW + std::string line_buffer(mLineBuffer.begin(), mLineBuffer.end()); + TNet::ParseHTKString(line_buffer, rName); + mLabels.Insert(rName, pos); + } + + return true; + } + else { + // we have been hashing all the way through so we know that if this is + // is the EOF, we are done hashing this stream + if (pos != static_cast<const std::streampos>(-1)) { + mIsHashed = true; + } + + // we are not in body state, so we just return false + return false; + } + } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>* + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + Close() + { + if (!mIsOpen) { + mIsEof = true; + return NULL; + } + else { + // if we try to close while in the body, we need to reach the end + if (mState == IN_BODY_STATE) { + while (mState == IN_BODY_STATE) { + FillLineBuffer(); + } + } + + // disable buffer mechanism + StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()), + &(mLineBuffer.front())); + + mIsEof = true; + mIsOpen = false; + + return this; + } + } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>* + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + Open(const std::string& rFileName) + { + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>* ret_val = NULL; + + // this behavior is compatible with ifstream + if (mIsOpen) { + Close(); + return NULL; + } + + // retreive position + std::streampos pos = mIStream.tellg(); + LabelRecord label_record; + + // for streams like stdin, pos will by definition be -1, so we can only + // rely on sequential access. At this place, we decide what to do + if ((-1 != pos) && (mLabels.Find(rFileName, label_record))) { + mIStream.seekg(label_record.mStreamPos); + mState = IN_TITLE_STATE; + + // we don't want the other stream to be bad, so we transfer the + // flagbits to this stream + if (!mIStream.good()) { + mIStream.clear(); + mIsOpen = false; + ret_val = NULL; + } + else { + mIsOpen = true; + mIsEof = false; + ret_val = this; + } + } + + // we don't have sequential stream and we didn't find the label, but + // we are hashed, so we can be sure, that we failed + else if ((-1 != pos) && mIsHashed) { + mIsOpen = false; + ret_val = NULL; + } + + // we either have sequential stream or didn't find anything, but we can + // still try to sequentially go and look for it + else { + bool found = false; + std::string aux_name; + std::string aux_name2; + + while ((!found) && JumpToNextDefinition(aux_name)) { + if (TNet::ProcessMask(rFileName, aux_name, aux_name2)) { + mIsOpen = true; + mIsEof = false; + found = true; + ret_val = this; + } + } + + if (!found) { + mIsOpen = false; + ret_val = NULL; + } + } + + return ret_val; + } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + typename _Traits::int_type + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + underflow() + { + // we don't do anything if EOF + if (mIsEof) { + StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()), + &(mLineBuffer.front())); + return _Traits::eof(); + } + + // read from buffer if we can + if (StreamBufType::gptr() && (StreamBufType::gptr() < StreamBufType::egptr())) { + return _Traits::not_eof(*StreamBufType::gptr()); + } + + // might happen that stream is in !good state + if (!mIStream.good()) { + mIsEof = true; + StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()), + &(mLineBuffer.front())); + return _Traits::eof(); + } + + // fill the line buffer and update my state + FillLineBuffer(); + + // if the whole line is just period or it's eof, declare EOF + if (mState == OUT_OF_BODY_STATE) { + mIsEof = true; + StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()), + &(mLineBuffer.front())); + return _Traits::eof(); + } + + // restore the buffer mechanism + StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()), + &(mLineBuffer.back()) + 1); + + return *StreamBufType::gptr(); + } + + + //**************************************************************************** + //**************************************************************************** + template< + typename _CharT, + typename _Traits, + typename _CharTA, + typename ByteT, + typename ByteAT + > + void + BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>:: + FillLineBuffer() + { + // reset line buffer + size_t capacity = mLineBuffer.capacity(); + mLineBuffer.clear(); + mLineBuffer.reserve(capacity); + + // read one line into buffer + int c; + while ((c = mIStream.get()) != '\n' && c != _Traits::eof()) { + mLineBuffer.push_back(char(c)); + } + + // we want to be able to pass last eol symbol + if (c == '\n') { + mLineBuffer.push_back(char(c)); + } + + // we will decide where we are + switch (mState) { + case IN_HEADER_STATE: + + case OUT_OF_BODY_STATE: + if (mLineBuffer[0] != '#') { + mState = IN_TITLE_STATE; + } + break; + + case IN_TITLE_STATE: + if (mLineBuffer[0] == '.' && (mLineBuffer.back() == '\n' || mIStream.eof())) { + mState = OUT_OF_BODY_STATE; + } + else { + mState = IN_BODY_STATE; + } + break; + + case IN_BODY_STATE: + // period or EOF will end the file + if (mLineBuffer[0] == '.' && (mLineBuffer.back() == '\n' || mIStream.eof())) { + mState = OUT_OF_BODY_STATE; + } + if (mLineBuffer.size() == 0) { + mState = OUT_OF_BODY_STATE; + } + break; + } + } +} // namespace TNet + + +#endif // STK_MlfStream_tcc diff --git a/tnet_io/KaldiLib/StkMatch.cc b/tnet_io/KaldiLib/StkMatch.cc new file mode 100644 index 0000000..4ff4b18 --- /dev/null +++ b/tnet_io/KaldiLib/StkMatch.cc @@ -0,0 +1,582 @@ +/* + EPSHeader + + File: filmatch.c + Author: J. Kercheval + Created: Thu, 03/14/1991 22:22:01 +*/ + +/* + EPSRevision History + O. Glembek Thu, 03/11/2005 01:58:00 Added Mask extraction support (char % does this) + J. Kercheval Wed, 02/20/1991 22:29:01 Released to Public Domain + J. Kercheval Fri, 02/22/1991 15:29:01 fix '\' bugs (two :( of them) + J. Kercheval Sun, 03/10/1991 19:31:29 add error return to matche() + J. Kercheval Sun, 03/10/1991 20:11:11 add is_valid_pattern code + J. Kercheval Sun, 03/10/1991 20:37:11 beef up main() + J. Kercheval Tue, 03/12/1991 22:25:10 Released as V1.1 to Public Domain + J. Kercheval Thu, 03/14/1991 22:22:25 remove '\' for DOS file parsing + J. Kercheval Thu, 03/28/1991 20:58:27 include filmatch.h +*/ + +/* + Wildcard Pattern Matching +*/ + + +#include "StkMatch.h" +#include "Common.h" + +namespace TNet +{ + //#define TEST + static int matche_after_star (register const char *pattern, register const char *text, register char *s); + // following function is not defined or used. + // static int fast_match_after_star (register const char *pattern, register const char *text); + + /*---------------------------------------------------------------------------- + * + * Return true if PATTERN has any special wildcard characters + * + ----------------------------------------------------------------------------*/ + + bool is_pattern (const char *p) + { + while ( *p ) { + switch ( *p++ ) { + case '?': + case '*': + case '%': + case '[': + return true; + } + } + return false; + } + + + /*---------------------------------------------------------------------------- + * + * Return true if PATTERN has is a well formed regular expression according + * to the above syntax + * + * error_type is a return code based on the type of pattern error. Zero is + * returned in error_type if the pattern is a valid one. error_type return + * values are as follows: + * + * PATTERN_VALID - pattern is well formed + * PATTERN_RANGE - [..] construct has a no end range in a '-' pair (ie [a-]) + * PATTERN_CLOSE - [..] construct has no end bracket (ie [abc-g ) + * PATTERN_EMPTY - [..] construct is empty (ie []) + * + ----------------------------------------------------------------------------*/ + + bool is_valid_pattern (const char *p, int *error_type) + { + + /* init error_type */ + *error_type = PATTERN_VALID; + + /* loop through pattern to EOS */ + while ( *p ) + { + /* determine pattern type */ + switch ( *p ) + { + /* the [..] construct must be well formed */ + case '[': + { + p++; + + /* if the next character is ']' then bad pattern */ + if ( *p == ']' ) { + *error_type = PATTERN_EMPTY; + return false; + } + + /* if end of pattern here then bad pattern */ + if ( !*p ) + { + *error_type = PATTERN_CLOSE; + return false; + } + + /* loop to end of [..] construct */ + while ( *p != ']' ) + { + /* check for literal escape */ + if ( *p == '\\' ) + { + p++; + + /* if end of pattern here then bad pattern */ + if ( !*p++ ) { + *error_type = PATTERN_ESC; + return false; + } + } + else + p++; + + /* if end of pattern here then bad pattern */ + if ( !*p ) + { + *error_type = PATTERN_CLOSE; + return false; + } + + /* if this a range */ + if ( *p == '-' ) + { + /* we must have an end of range */ + if ( !*++p || *p == ']' ) + { + *error_type = PATTERN_RANGE; + return false; + } + else + { + + /* check for literal escape */ + if ( *p == '\\' ) + p++; + + /* if end of pattern here then bad pattern */ + if ( !*p++ ) + { + *error_type = PATTERN_ESC; + return false; + } + } + } + } + break; + } //case '[': + + + /* all other characters are valid pattern elements */ + case '*': + case '?': + case '%': + default: + p++; /* "normal" character */ + break; + } // switch ( *p ) + } // while ( *p ) + + return true; + } //bool is_valid_pattern (const char *p, int *error_type) + + + /*---------------------------------------------------------------------------- + * + * Match the pattern PATTERN against the string TEXT; + * + * returns MATCH_VALID if pattern matches, or an errorcode as follows + * otherwise: + * + * MATCH_PATTERN - bad pattern + * MATCH_RANGE - match failure on [..] construct + * MATCH_ABORT - premature end of text string + * MATCH_END - premature end of pattern string + * MATCH_VALID - valid match + * + * + * A match means the entire string TEXT is used up in matching. + * + * In the pattern string: + * `*' matches any sequence of characters (zero or more) + * `?' matches any character + * `%' matches any character and stores it in the s string + * [SET] matches any character in the specified set, + * [!SET] or [^SET] matches any character not in the specified set. + * \ is allowed within a set to escape a character like ']' or '-' + * + * A set is composed of characters or ranges; a range looks like + * character hyphen character (as in 0-9 or A-Z). [0-9a-zA-Z_] is the + * minimal set of characters allowed in the [..] pattern construct. + * Other characters are allowed (ie. 8 bit characters) if your system + * will support them. + * + * To suppress the special syntactic significance of any of `[]*?%!^-\', + * within a [..] construct and match the character exactly, precede it + * with a `\'. + * + ----------------------------------------------------------------------------*/ + + int matche ( register const char *p, register const char *t, register char *s ) + { + register char range_start, range_end; /* start and end in range */ + + bool invert; /* is this [..] or [!..] */ + bool member_match; /* have I matched the [..] construct? */ + bool loop; /* should I terminate? */ + + for ( ; *p; p++, t++ ) { + + /* if this is the end of the text then this is the end of the match */ + if (!*t) { + return ( *p == '*' && *++p == '\0' ) ? MATCH_VALID : MATCH_ABORT; + } + + /* determine and react to pattern type */ + switch ( *p ) { + + /* single any character match */ + case '?': + break; + + /* single any character match, with extraction*/ + case '%': { + *s++ = *t; + *s = '\0'; + break; + } + + /* multiple any character match */ + case '*': + return matche_after_star (p, t, s); + + /* [..] construct, single member/exclusion character match */ + case '[': { + /* move to beginning of range */ + p++; + + /* check if this is a member match or exclusion match */ + invert = false; + if ( *p == '!' || *p == '^') { + invert = true; + p++; + } + + /* if closing bracket here or at range start then we have a + malformed pattern */ + if ( *p == ']' ) { + return MATCH_PATTERN; + } + + member_match = false; + loop = true; + + while ( loop ) { + + /* if end of construct then loop is done */ + if (*p == ']') { + loop = false; + continue; + } + + /* matching a '!', '^', '-', '\' or a ']' */ + if ( *p == '\\' ) { + range_start = range_end = *++p; + } + else { + range_start = range_end = *p; + } + + /* if end of pattern then bad pattern (Missing ']') */ + if (!*p) + return MATCH_PATTERN; + + /* check for range bar */ + if (*++p == '-') { + + /* get the range end */ + range_end = *++p; + + /* if end of pattern or construct then bad pattern */ + if (range_end == '\0' || range_end == ']') + return MATCH_PATTERN; + + /* special character range end */ + if (range_end == '\\') { + range_end = *++p; + + /* if end of text then we have a bad pattern */ + if (!range_end) + return MATCH_PATTERN; + } + + /* move just beyond this range */ + p++; + } + + /* if the text character is in range then match found. + make sure the range letters have the proper + relationship to one another before comparison */ + if ( range_start < range_end ) { + if (*t >= range_start && *t <= range_end) { + member_match = true; + loop = false; + } + } + else { + if (*t >= range_end && *t <= range_start) { + member_match = true; + loop = false; + } + } + } + + /* if there was a match in an exclusion set then no match */ + /* if there was no match in a member set then no match */ + if ((invert && member_match) || + !(invert || member_match)) + return MATCH_RANGE; + + /* if this is not an exclusion then skip the rest of the [...] + construct that already matched. */ + if (member_match) { + while (*p != ']') { + + /* bad pattern (Missing ']') */ + if (!*p) + return MATCH_PATTERN; + + /* skip exact match */ + if (*p == '\\') { + p++; + + /* if end of text then we have a bad pattern */ + if (!*p) + return MATCH_PATTERN; + } + + /* move to next pattern char */ + p++; + } + } + + break; + } // case ']' + + /* must match this character exactly */ + default: + if (*p != *t) + return MATCH_LITERAL; + } + } + + //*s = '\0'; + /* if end of text not reached then the pattern fails */ + if ( *t ) + return MATCH_END; + else + return MATCH_VALID; + } + + + /*---------------------------------------------------------------------------- + * + * recursively call matche() with final segment of PATTERN and of TEXT. + * + ----------------------------------------------------------------------------*/ + + static int matche_after_star (register const char *p, register const char *t, register char *s) + { + register int match = 0; + register char nextp; + + /* pass over existing ? and * in pattern */ + while ( *p == '?' || *p == '%' || *p == '*' ) { + + /* take one char for each ? and + */ + if ( *p == '?') { + + /* if end of text then no match */ + if ( !*t++ ) { + return MATCH_ABORT; + } + } + + if ( *p == '%') { + *s++ = *t; + *s = '\0'; + /* if end of text then no match */ + if ( !*t++ ) { + return MATCH_ABORT; + } + } + + /* move to next char in pattern */ + p++; + } + + /* if end of pattern we have matched regardless of text left */ + if ( !*p ) { + return MATCH_VALID; + } + + /* get the next character to match which must be a literal or '[' */ + nextp = *p; + + /* Continue until we run out of text or definite result seen */ + do { + + /* a precondition for matching is that the next character + in the pattern match the next character in the text or that + the next pattern char is the beginning of a range. Increment + text pointer as we go here */ + if ( nextp == *t || nextp == '[' ) { + match = matche(p, t, s); + } + + /* if the end of text is reached then no match */ + if ( !*t++ ) match = MATCH_ABORT; + + } while ( match != MATCH_VALID && + match != MATCH_ABORT && + match != MATCH_PATTERN); + + /* return result */ + return match; + } + + + /*---------------------------------------------------------------------------- + * + * match() is a shell to matche() to return only bool values. + * + ----------------------------------------------------------------------------*/ + + bool match(const char *p, const char *t, char *s) + { + int error_type; + error_type = matche(p,t,s); + return (error_type != MATCH_VALID ) ? false : true; + } + + + //*************************************************************************** + //*************************************************************************** + bool + ProcessMask(const std::string & rString, + const std::string & rWildcard, + std::string & rSubstr) + { + char * substr; + int percent_count = 0; + int ret ; + size_t pos = 0; + + // let's find how many % to allocate enough space for the return substring + while ((pos = rWildcard.find('%', pos)) != rWildcard.npos) + { + percent_count++; + pos++; + } + + // allocate space for the substring + substr = new char[percent_count + 1]; + substr[percent_count] = 0; + substr[0] = '\0'; + + // optionally prepend '*/' to wildcard + std::string wildcard(rWildcard); + if(wildcard[0] != '*') { + wildcard = "*/" + wildcard; + } + + //optionally prepend '/' to string + std::string string1(rString); + if(string1[0] != '/') { + string1 = "/" + string1; + } + + // parse the string + if (0 != (ret = match(wildcard.c_str(), string1.c_str(), substr))) + { + rSubstr = substr; + } + delete[] substr; + return ret; + } // ProcessMask +} + + +#ifdef TEST + +/* +* This test main expects as first arg the pattern and as second arg +* the match string. Output is yaeh or nay on match. If nay on +* match then the error code is parsed and written. +*/ + +#include <stdio.h> + +int main(int argc, char *argv[]) +{ + int error; + int is_valid_error; + + char * tmp = argv[0]; + int i = 0; + for (; *tmp; tmp++) + if (*tmp=='%') i++; + + char s[i+1]; + + + if (argc != 3) { + printf("Usage: MATCH Pattern Text\n"); + } + else { + printf("Pattern: %s\n", argv[1]); + printf("Text : %s\n", argv[2]); + + if (!is_pattern(argv[1])) { + printf(" First Argument Is Not A Pattern\n"); + } + else { + match(argv[1],argv[2], s) ? printf("true") : printf("false"); + error = matche(argv[1],argv[2], s); + is_valid_pattern(argv[1],&is_valid_error); + + switch ( error ) { + case MATCH_VALID: + printf(" Match Successful"); + if (is_valid_error != PATTERN_VALID) + printf(" -- is_valid_pattern() is complaining\n"); + else + printf("\n"); + printf("%s\n", s); + + break; + case MATCH_RANGE: + printf(" Match Failed on [..]\n"); + break; + case MATCH_ABORT: + printf(" Match Failed on Early Text Termination\n"); + break; + case MATCH_END: + printf(" Match Failed on Early Pattern Termination\n"); + break; + case MATCH_PATTERN: + switch ( is_valid_error ) { + case PATTERN_VALID: + printf(" Internal Disagreement On Pattern\n"); + break; + case PATTERN_RANGE: + printf(" No End of Range in [..] Construct\n"); + break; + case PATTERN_CLOSE: + printf(" [..] Construct is Open\n"); + break; + case PATTERN_EMPTY: + printf(" [..] Construct is Empty\n"); + break; + default: + printf(" Internal Error in is_valid_pattern()\n"); + } + break; + default: + printf(" Internal Error in matche()\n"); + break; + } + } + + } + return(0); +} + +#endif diff --git a/tnet_io/KaldiLib/StkMatch.h b/tnet_io/KaldiLib/StkMatch.h new file mode 100644 index 0000000..42c6b97 --- /dev/null +++ b/tnet_io/KaldiLib/StkMatch.h @@ -0,0 +1,123 @@ +#ifndef TNet_StkMatch_h +#define TNet_StkMatch_h + +#include <string> +namespace TNet +{ + /* + EPSHeader + + File: filmatch.h + Author: J. Kercheval + Created: Thu, 03/14/1991 22:24:34 + */ + + /* + EPSRevision History + O. Glembek Thu, 03/11/2005 01:58:00 Added Mask extraction support (char % does this) + J. Kercheval Wed, 02/20/1991 22:28:37 Released to Public Domain + J. Kercheval Sun, 03/10/1991 18:02:56 add is_valid_pattern + J. Kercheval Sun, 03/10/1991 18:25:48 add error_type in is_valid_pattern + J. Kercheval Sun, 03/10/1991 18:47:47 error return from matche() + J. Kercheval Tue, 03/12/1991 22:24:49 Released as V1.1 to Public Domain + J. Kercheval Thu, 03/14/1991 22:25:00 remove '\' for DOS file matching + J. Kercheval Thu, 03/28/1991 21:03:59 add in PATTERN_ESC & MATCH_LITERAL + */ + + /* + Wildcard Pattern Matching + */ + + + /* match defines */ +#define MATCH_PATTERN 6 /* bad pattern */ +#define MATCH_LITERAL 5 /* match failure on literal match */ +#define MATCH_RANGE 4 /* match failure on [..] construct */ +#define MATCH_ABORT 3 /* premature end of text string */ +#define MATCH_END 2 /* premature end of pattern string */ +#define MATCH_VALID 1 /* valid match */ + + /* pattern defines */ +#define PATTERN_VALID 0 /* valid pattern */ +#define PATTERN_ESC -1 /* literal escape at end of pattern */ +#define PATTERN_RANGE -2 /* malformed range in [..] construct */ +#define PATTERN_CLOSE -3 /* no end bracket in [..] construct */ +#define PATTERN_EMPTY -4 /* [..] contstruct is empty */ + + + /*---------------------------------------------------------------------------- + * + * Match the pattern PATTERN against the string TEXT; + * + * match() returns TRUE if pattern matches, FALSE otherwise. + * matche() returns MATCH_VALID if pattern matches, or an errorcode + * as follows otherwise: + * + * MATCH_PATTERN - bad pattern + * MATCH_RANGE - match failure on [..] construct + * MATCH_ABORT - premature end of text string + * MATCH_END - premature end of pattern string + * MATCH_VALID - valid match + * + * + * A match means the entire string TEXT is used up in matching. + * + * In the pattern string: + * `*' matches any sequence of characters (zero or more) + * `?' matches any character + * [SET] matches any character in the specified set, + * [!SET] or [^SET] matches any character not in the specified set. + * + * A set is composed of characters or ranges; a range looks like + * character hyphen character (as in 0-9 or A-Z). [0-9a-zA-Z_] is the + * minimal set of characters allowed in the [..] pattern construct. + * Other characters are allowed (ie. 8 bit characters) if your system + * will support them. + * + * To suppress the special syntactic significance of any of `[]*?!^-\', + * in a [..] construct and match the character exactly, precede it + * with a `\'. + * + ----------------------------------------------------------------------------*/ + bool + match (const char *pattern, const char *text, char *s); + + int + matche(register const char *pattern, register const char *text, register char *s); + + + /*---------------------------------------------------------------------------- + * + * Return TRUE if PATTERN has any special wildcard characters + * + ----------------------------------------------------------------------------*/ + bool + is_pattern (const char *pattern); + + + /** -------------------------------------------------------------------------- + * + * Return TRUE if PATTERN has is a well formed regular expression according + * to the above syntax + * + * error_type is a return code based on the type of pattern error. Zero is + * returned in error_type if the pattern is a valid one. error_type return + * values are as follows: + * + * PATTERN_VALID - pattern is well formed + * PATTERN_RANGE - [..] construct has a no end range in a '-' pair (ie [a-]) + * PATTERN_CLOSE - [..] construct has no end bracket (ie [abc-g ) + * PATTERN_EMPTY - [..] construct is empty (ie []) + * -------------------------------------------------------------------------- + **/ + bool + is_valid_pattern (const char *pattern, int *error_type); + + + //**************************************************************************** + //**************************************************************************** + bool + ProcessMask(const std::string & rString, const std::string & rWildcard, + std::string & rSubstr); +} +#endif diff --git a/tnet_io/KaldiLib/StkStream.h b/tnet_io/KaldiLib/StkStream.h new file mode 100644 index 0000000..ca8de30 --- /dev/null +++ b/tnet_io/KaldiLib/StkStream.h @@ -0,0 +1,526 @@ + + +/** @file stkstream.h + * This is an TNet C++ Library header. + */ + + +#ifndef TNet_StkStream_h +#define TNet_StkStream_h + +#include <fstream> +#include <string> +#include <vector> +#include <list> +#include <stdexcept> + +#pragma GCC system_header + + +//extern const char * gpFilterWldcrd; + +namespace TNet +{ + + /** + * @brief Expands a filter command into a runnable form + * + * This function replaces all occurances of *filter_wldcard in *command by + * *filename + */ + //char * ExpandFilterCommand(const char *command, const char *filename); + + /** + * @brief Provides a layer of compatibility for C/POSIX. + * + * This GNU extension provides extensions for working with standard C + * FILE*'s and POSIX file descriptors. It must be instantiated by the + * user with the type of character used in the file stream, e.g., + * basic_stkbuf<char>. + */ + template< + typename _CharT, + typename _Traits = std::char_traits<_CharT> + > + class basic_stkbuf : public std::basic_filebuf<_CharT, _Traits> + { + public: + + typedef basic_stkbuf<_CharT, _Traits> this_type; + + // Types: + typedef _CharT char_type; + typedef _Traits traits_type; + + typedef typename traits_type::int_type int_type; + typedef typename traits_type::pos_type pos_type; + typedef typename traits_type::off_type off_type; + typedef std::size_t size_t; + + public: + + /// @{ + /// Type of streambuffer + static const unsigned int t_undef = 0; ///< undefined + static const unsigned int t_file = 1; ///< file stream + static const unsigned int t_pipe = 2; ///< pipe + static const unsigned int t_filter = 4; ///< filter + static const unsigned int t_stdio = 8; ///< standard input/output + /// @} + + public: + + /** + * deferred initialization + */ + basic_stkbuf() : std::basic_filebuf<_CharT, _Traits>(), + mFilename(""), mpFilePtr(0), mStreamType(t_undef){} + + /** + * @brief Opens a stream. + * @param fName The name of the file. + * @param m The open mode flags. + * @param pFilter The pFilter command to use + * @return @c this on success, NULL on failure + * + * If a file is already open, this function immediately fails. + * Otherwise it tries to open the file named @a s using the flags + * given in @a mode. + * + * [Table 92 gives the relation between openmode combinations and the + * equivalent fopen() flags, but the table has not been copied yet.] + */ + basic_stkbuf(const char* fName, std::ios_base::openmode m, const char* pFilter=""); + + + /** + * @return The underlying FILE*. + * + * This function can be used to access the underlying "C" file pointer. + * Note that there is no way for the library to track what you do + * with the file, so be careful. + */ + std::__c_file* + file() { return this->_M_file.file(); } + + + /** + * @return The underlying FILE*. + * + * This function can be used to access the underlying "C" file pointer. + * Note that there is no way for the library to track what you do + * with the file, so be careful. + */ + std::__c_file* + fp() { return this->_M_file.file(); } + + + /** + * @brief Opens an external file. + * @param fName The name of the file. + * @param m The open mode flags. + * @param pFilter The pFilter command to use + * @return @c this on success, NULL on failure + * + * If a file is already open, this function immediately fails. + * Otherwise it tries to open the file named @a s using the flags + * given in @a mode. + * + * [Table 92 gives the relation between openmode combinations and the + * equivalent fopen() flags, but the table has not been copied yet.] + */ + this_type* + open(const char* pFName, std::ios_base::openmode m, const char* pFilter=""); + + /** + * @brief Closes the currently associated file. + * @return @c this on success, NULL on failure + * + * If no file is currently open, this function immediately fails. + * + * If a "put buffer area" exists, @c overflow(eof) is called to flush + * all the characters. The file is then closed. + * + * If any operations fail, this function also fails. + */ + this_type* + close(); + + /** + * Closes the external data stream if the file descriptor constructor + * was used. + */ + virtual + ~basic_stkbuf() + {close();}; + + /// Returns the file name + const std::string + name() const + {return mFilename;} + + + private: + /// converts the ios::xxx mode to stdio style + static void open_mode(std::ios_base::openmode __mode, int&, int&, char* __c_mode); + + /** + * @param __f An open @c FILE*. + * @param __mode Same meaning as in a standard filebuf. + * @param __size Optimal or preferred size of internal buffer, in chars. + * Defaults to system's @c BUFSIZ. + * + * This method associates a file stream buffer with an open + * C @c FILE*. The @c FILE* will not be automatically closed when the + * basic_stkbuf is closed/destroyed. It is equivalent to one of the constructors + * of the stdio_filebuf class defined in GNU ISO C++ ext/stdio_filebuf.h + */ + void superopen(std::__c_file* __f, std::ios_base::openmode __mode, + size_t __size = static_cast<size_t>(BUFSIZ)); + + + private: + /// Holds the full file name + std::string mFilename; + + std::ios_base::openmode mMode; + + /// Holds a pointer to the main FILE structure + FILE * mpFilePtr; + + /// tells what kind of stream we use (stdio, file, pipe) + unsigned int mStreamType; + + }; + + + + /** + * @brief This extension wraps stkbuf stream buffer into the standard ios class. + * + * This class is inherited by (i/o)stkstream classes which make explicit use of + * the custom stream buffer + */ + template< + typename _CharT, + typename _Traits = std::char_traits<_CharT> + > + class BasicStkIos + : virtual public std::basic_ios<_CharT, _Traits> + { + public: + typedef basic_stkbuf <_CharT,_Traits> StkBufType; + + BasicStkIos() + : mBuf() + { this->init(&mBuf) ;}; + + BasicStkIos(const char* fName, std::ios::openmode m, const char* pFilter) + : mBuf(fName, m, pFilter) + { this->init(&mBuf) ; } + + StkBufType* + rdbuf() + { return &mBuf; } + + protected: + StkBufType mBuf; + }; + + + /** + * @brief Controlling input for files. + * + * This class supports reading from named files, using the inherited + * functions from std::istream. To control the associated + * sequence, an instance of std::stkbuf is used. + */ + template< + typename _CharT, + typename _Traits = std::char_traits<_CharT> + > + class BasicIStkStream + : public BasicStkIos<_CharT, _Traits>, + public std::basic_istream<_CharT, _Traits> + { + public: + typedef BasicStkIos<_CharT, _Traits> BasicStkIosType; + typedef std::basic_istream<_CharT,_Traits> IStreamType; + + + // Constructors: + /** + * @brief Default constructor. + * + * Initializes @c mBuf using its default constructor, and passes + * @c &sb to the base class initializer. Does not open any files + * (you haven't given it a filename to open). + */ + BasicIStkStream() + : BasicStkIosType(), + IStreamType(BasicStkIosType::rdbuf()) + {}; + + /** + * @brief Create an input file stream. + * @param fName String specifying the filename. + * @param m Open file in specified mode (see std::ios_base). + * @param pFilter String specifying pFilter command to use on fName + * + * @c ios_base::in is automatically included in + * @a m. + * + * Tip: When using std::string to hold the filename, you must use + * .c_str() before passing it to this constructor. + */ + BasicIStkStream(const char* pFName, std::ios::openmode m=std::ios::out, const char* pFilter="") + : BasicStkIosType(), + IStreamType(BasicStkIosType::rdbuf()) + {this->open(pFName, std::ios::in, pFilter);} + + ~BasicIStkStream() + { + this->close(); + } + + /** + * @brief Opens an external file. + * @param s The name of the file. + * @param mode The open mode flags. + * @param pFilter The pFilter command to use + * + * Calls @c std::basic_filebuf::open(s,mode|in). If that function + * fails, @c failbit is set in the stream's error state. + * + * Tip: When using std::string to hold the filename, you must use + * .c_str() before passing it to this constructor. + */ + void open(const char* pFName, std::ios::openmode m=std::ios::in, const char* pFilter = "") + { + if (!BasicStkIosType::mBuf.open(pFName, m | std::ios_base::in, pFilter)) { + this->setstate(std::ios_base::failbit); + } + else { + // Closing an fstream should clear error state + BasicStkIosType::clear(); + } + } + + /** + * @brief Returns true if the external file is open. + */ + bool is_open() const {return BasicStkIosType::mBuf.is_open();} + + + /** + * @brief Closes the stream + */ + void close() {BasicStkIosType::mBuf.close();} + + /** + * @brief Returns the filename + */ + const std::string name() const {return BasicStkIosType::mBuf.name();} + + /// Returns a pointer to the main FILE structure + std::__c_file* + file() {return BasicStkIosType::mBuf.file();} + + /// Returns a pointer to the main FILE structure + std::__c_file* + fp() {return BasicStkIosType::mBuf.fp();} + + // /** + // * @brief Reads a single line + // * + // * This is a specialized function as std::getline does not provide a way to + // * read multiple end-of-line symbols (we need both '\n' and EOF to delimit + // * the line) + // */ + // void + // GetLine(string& rLine); + + }; // class BasicIStkStream + + + /** + * @brief Controlling output for files. + * + * This class supports reading from named files, using the inherited + * functions from std::ostream. To control the associated + * sequence, an instance of TNet::stkbuf is used. + */ + template< + typename _CharT, + typename _Traits = std::char_traits<_CharT> + > + class BasicOStkStream + : public BasicStkIos<_CharT, _Traits>, + public std::basic_ostream<_CharT, _Traits> + { + public: + typedef BasicStkIos<_CharT, _Traits> BasicStkIosType; + typedef std::basic_ostream<_CharT,_Traits> OStreamType; + + // Constructors: + /** + * @brief Default constructor. + * + * Initializes @c sb using its default constructor, and passes + * @c &sb to the base class initializer. Does not open any files + * (you haven't given it a filename to open). + */ + BasicOStkStream() + : BasicStkIosType(), + OStreamType(BasicStkIosType::rdbuf()) + {}; + + /** + * @brief Create an output file stream. + * @param fName String specifying the filename. + * @param m Open file in specified mode (see std::ios_base). + * @param pFilter String specifying pFilter command to use on fName + * + * @c ios_base::out|ios_base::trunc is automatically included in + * @a mode. + * + * Tip: When using std::string to hold the filename, you must use + * .c_str() before passing it to this constructor. + */ + BasicOStkStream(const char* pFName, std::ios::openmode m=std::ios::out, const char* pFilter="") + : BasicStkIosType(), + OStreamType(BasicStkIosType::rdbuf()) + {this->open(pFName, std::ios::out, pFilter);} + + /** + * @brief Opens an external file. + * @param fName The name of the file. + * @param m The open mode flags. + * @param pFilter String specifying pFilter command to use on fName + * + * Calls @c std::basic_filebuf::open(s,mode|out). If that function + * fails, @c failbit is set in the stream's error state. + * + * Tip: When using std::string to hold the filename, you must use + * .c_str() before passing it to this constructor. + */ + void open(const char* pFName, std::ios::openmode m=std::ios::out, const char* pFilter="") + { + if (!BasicStkIosType::mBuf.open(pFName, m | std::ios_base::out, pFilter)) + this->setstate(std::ios_base::failbit); + else + // Closing an fstream should clear error state + this->clear(); + } + + /** + * @brief Returns true if the external file is open. + */ + bool is_open() const + { return BasicStkIosType::mBuf.is_open();} + + /** + * @brief Closes the stream + */ + void close() + { BasicStkIosType::mBuf.close();} + + /** + * @brief Returns the filename + */ + const std::string name() const + { return BasicStkIosType::mBuf.name();} + + /// Returns a pointer to the main FILE structure + std::__c_file* + file() + { return BasicStkIosType::mBuf.file();} + + /// Returns a pointer to the main FILE structure + std::__c_file* + fp() + { return BasicStkIosType::mBuf.fp();} + + }; // class BasicOStkStream + + + /** + * We define some implicit stkbuf class + */ + ///@{ +#ifndef _GLIBCPP_USE_WCHAR_T + typedef BasicOStkStream<char> OStkStream; + typedef BasicOStkStream<wchar_t> WOStkStream; + typedef BasicIStkStream<char> IStkStream; + typedef BasicIStkStream<wchar_t> WIStkStream; +#else + typedef BasicOStkStream<char> WOStkStream; + typedef BasicOStkStream<wchar_t> WOStkStream; + typedef BasicIStkStream<char> WIStkStream; + typedef BasicIStkStream<wchar_t> WIStkStream; +#endif + /// @} + + /* + template<class T,class char_type> inline + BasicOStkStream<char_type>& operator << (BasicOStkStream<char_type> &ostream, const std::vector<T> &vec){ + ostream << vec.size() << std::endl; + for(size_t i=0;i<vec.size();i++) ostream << vec[i]; + return ostream; + } + + template<class T,class char_type> inline BasicIStkStream<char_type> &operator >> (BasicIStkStream<char_type> &istream, std::vector<T> &vec){ + size_t sz; + istream >> sz; if(!istream.good()){ throw std::runtime_error(std::string("Error reading to vector of [something]: stream bad\n")); } + int ch = istream.get(); if(ch!='\n' || !istream.good()){ throw std::runtime_error(std::string("Expecting newline after vector size, got " + (std::string)(char)ch));} // TODO: This code may not be right for wchar. + vec.resize(sz); + for(size_t i=0;i<vec.size();i++) istream >> vec[i]; + return istream; + }*/ + + template<class T> inline + std::ostream & operator << (std::ostream &ostream, const std::vector<T> &vec){ + ostream << vec.size() << std::endl; + for(size_t i=0;i<vec.size();i++) ostream << vec[i] << "\n"; // '\n' is necessary in case item is atomic e.g. a number. + return ostream; + } + + template<class T> inline std::istream& operator >> (std::istream &istream, std::vector<T> &vec){ + size_t sz; + istream >> sz; if(!istream.good()){ throw std::runtime_error(std::string("Error reading to vector of [something]: stream bad\n")); } + // int ch = istream.get(); if(ch!='\n' || !istream.good()){ throw std::runtime_error(std::string("Expecting newline after vector size\n")); // TODO: This code may not be right for wchar. + vec.resize(sz); + for(size_t i=0;i<vec.size();i++) istream >> vec[i]; + return istream; + } + + template<class T> inline + std::ostream & operator << (std::ostream &ostream, const std::list<T> &lst){ + ostream << lst.size() << std::endl; + typename std::list<T>::iterator it; + for(it = lst.begin(); it != lst.end(); it++) + ostream << *it << "\n"; // '\n' is necessary in case item is atomic e.g. a number. + return ostream; + } + + template<class T> inline std::istream& operator >> (std::istream &istream, std::list<T> &lst){ + size_t sz; + istream >> sz; if(!istream.good()){ throw std::runtime_error(std::string("Error reading to list of [something]: stream bad\n")); } + lst.resize(sz); + typename std::list<T>::iterator it; + for(it = lst.begin(); it != lst.end(); it++) + istream >> *it; + return istream; + } + +}; // namespace TNet + + +using TNet::operator >>; +using TNet::operator <<; + + +# include "StkStream.tcc" + +// TNet_StkStream_h +#endif diff --git a/tnet_io/KaldiLib/StkStream.tcc b/tnet_io/KaldiLib/StkStream.tcc new file mode 100644 index 0000000..e3de1ae --- /dev/null +++ b/tnet_io/KaldiLib/StkStream.tcc @@ -0,0 +1,228 @@ +#ifndef TNet_StkStream_tcc +#define TNet_StkStream_tcc + +#include <cstring> +#include <iostream> + +#include "Common.h" + +#pragma GCC system_header + +namespace TNet +{ + + //****************************************************************************** + template< + typename _CharT, + typename _Traits + > + basic_stkbuf<_CharT, _Traits> * + basic_stkbuf<_CharT, _Traits>:: + close(void) + { + // we only want to close an opened file + if (this->is_open()) + { + // we want to call the parent close() procedure + std::basic_filebuf<_CharT, _Traits>::close(); + + // and for different stream type we perform different closing + if (mStreamType == basic_stkbuf::t_file) + { + fclose(mpFilePtr); + } + else if (mStreamType == basic_stkbuf::t_pipe) + { + pclose(mpFilePtr); + } + else if (mStreamType == basic_stkbuf::t_stdio) + { + + } + + mpFilePtr = NULL; + mFilename = ""; + mMode = std::ios_base::openmode(0); + mStreamType = basic_stkbuf::t_undef; + return this; + } + else + return 0; + } + + + template< + typename _CharT, + typename _Traits + > + void + basic_stkbuf<_CharT, _Traits>:: + open_mode(std::ios_base::openmode __mode, int&, int&, char* __c_mode) + { + bool __testb = __mode & std::ios_base::binary; + bool __testi = __mode & std::ios_base::in; + bool __testo = __mode & std::ios_base::out; + bool __testt = __mode & std::ios_base::trunc; + bool __testa = __mode & std::ios_base::app; + + if (!__testi && __testo && !__testt && !__testa) + strcpy(__c_mode, "w"); + if (!__testi && __testo && !__testt && __testa) + strcpy(__c_mode, "a"); + if (!__testi && __testo && __testt && !__testa) + strcpy(__c_mode, "w"); + if (__testi && !__testo && !__testt && !__testa) + strcpy(__c_mode, "r"); + if (__testi && __testo && !__testt && !__testa) + strcpy(__c_mode, "r+"); + if (__testi && __testo && __testt && !__testa) + strcpy(__c_mode, "w+"); + if (__testb) + strcat(__c_mode, "b"); + } + + + //****************************************************************************** + template< + typename _CharT, + typename _Traits + > + basic_stkbuf<_CharT, _Traits> * + basic_stkbuf<_CharT, _Traits>:: + open(const char* pFName, std::ios::openmode m, const char* pFilter) + { + basic_stkbuf<_CharT, _Traits>* p_ret = NULL; + + if (NULL == pFName) + return NULL; + + // we need to assure, that the stream is not open + if (!this->is_open()) + { + char mstr[4] = {'\0', '\0', '\0', '\0'}; + int __p_mode = 0; + int __rw_mode = 0; + + // now we decide, what kind of file we open + if (!strcmp(pFName,"-")) + { + if ((m & std::ios::in) && !(m & std::ios::out)) + { + mpFilePtr = stdin; + mMode = std::ios::in; + mFilename = pFName; + mStreamType = t_stdio; + p_ret = this; + } + else if ((m & std::ios::out) && !(m & std::ios::in)) + { + mpFilePtr = stdout; + mMode = std::ios::out; + mFilename = pFName; + mStreamType = t_stdio; + p_ret = this; + } + else + p_ret = NULL; + } + else if ( pFName[0] == '|' ) + { + const char* command = pFName + 1; + + if ((m & std::ios::in) && !(m & std::ios::out)) m = std::ios::in; + else if ((m & std::ios::out) && !(m & std::ios::in)) m = std::ios::out; + else return NULL; + + // we need to make some conversion + // iostream -> stdio open mode string + this->open_mode(m, __p_mode, __rw_mode, mstr); + + if ((mpFilePtr = popen(command, mstr))) + { + mFilename = command; + mMode = m; + mStreamType = t_pipe; + p_ret = this; + } + else + p_ret = 0; + } + else + { + // maybe we have a filter specified + if ( pFilter + && ('\0' != pFilter[0])) + { + char* command = ExpandHtkFilterCmd(pFilter, pFName, "$"); + + if ((m & std::ios::in) && !(m & std::ios::out)) m = std::ios::in; + else if ((m & std::ios::out) && !(m & std::ios::in)) m = std::ios::out; + else return NULL; + + // we need to make some conversion + // iostream -> stdio open mode string + this->open_mode(m, __p_mode, __rw_mode, mstr); + + if ((mpFilePtr = popen(command, mstr))) + { + mFilename = pFName; + mMode = m; + mStreamType = t_pipe; + p_ret = this; + } + else + p_ret = 0; + } + else // if (!filter.empty()) + { + // we need to make some conversion + // iostream -> stdio open mode string + this->open_mode(m, __p_mode, __rw_mode, mstr); + + if ((mpFilePtr = fopen(pFName, mstr))) + { + mFilename = pFName; + mMode = m; + mStreamType = t_file; + p_ret = this; + } + else { + p_ret = NULL; + } + } + } + + // here we perform what the stdio_filebuf would do + if (p_ret) { + superopen(mpFilePtr, m); + } + } //if (!isopen) + + return p_ret; + } + + //****************************************************************************** + template< + typename _CharT, + typename _Traits + > + void + basic_stkbuf<_CharT, _Traits>:: + superopen(std::__c_file* __f, std::ios_base::openmode __mode, + size_t __size) + { + this->_M_file.sys_open(__f, __mode); + if (this->is_open()) + { + this->_M_mode = __mode; + this->_M_buf_size = __size; + this->_M_allocate_internal_buffer(); + this->_M_reading = false; + this->_M_writing = false; + this->_M_set_buffer(-1); + } + } +} + +// TNet_StkStream_tcc +#endif diff --git a/tnet_io/KaldiLib/Timer.cc b/tnet_io/KaldiLib/Timer.cc new file mode 100644 index 0000000..692969b --- /dev/null +++ b/tnet_io/KaldiLib/Timer.cc @@ -0,0 +1,5 @@ +#include "Timer.h" + +/* +TNet::Timer gTimer; +*/ diff --git a/tnet_io/KaldiLib/Timer.h b/tnet_io/KaldiLib/Timer.h new file mode 100644 index 0000000..b220b93 --- /dev/null +++ b/tnet_io/KaldiLib/Timer.h @@ -0,0 +1,103 @@ +#ifndef Timer_h +#define Timer_h + +#include "Error.h" +#include <sstream> + + + +#if defined(_WIN32) || defined(MINGW) + +# include <windows.h> + +namespace TNet +{ + class Timer { + public: + void + Start(void) + { + static int first = 1; + + if(first) { + QueryPerformanceFrequency(&mFreq); + first = 0; + } + QueryPerformanceCounter(&mTStart); + } + + void + End(void) + { QueryPerformanceCounter(&mTEnd); } + + double + Val() + { + return ((double)mTEnd.QuadPart - (double)mTStart.QuadPart) / + ((double)mFreq.QuadPart); + } + + private: + LARGE_INTEGER mTStart; + LARGE_INTEGER mTEnd; + LARGE_INTEGER mFreq; + }; +} + +#else + +# include <sys/time.h> +# include <unistd.h> + +namespace TNet +{ + class Timer + { + public: + void + Start() + { gettimeofday(&this->mTStart, &mTz); } + + void + End() + { gettimeofday(&mTEnd,&mTz); } + + double + Val() + { + double t1, t2; + + t1 = (double)mTStart.tv_sec + (double)mTStart.tv_usec/(1000*1000); + t2 = (double)mTEnd.tv_sec + (double)mTEnd.tv_usec/(1000*1000); + return t2-t1; + } + + private: + struct timeval mTStart; + struct timeval mTEnd; + struct timezone mTz; + }; +} + +#endif + + + + + + + +/////////////////////////////////////////////////////////////// +// Macros for adding the time intervals to time accumulator +#if PROFILING==1 +# define TIMER_START(timer) timer.Start() +# define TIMER_END(timer,sum) timer.End(); sum += timer.Val() +#else +# define TIMER_START(timer) +# define TIMER_END(timer,sum) +#endif + +#endif + + + diff --git a/tnet_io/KaldiLib/Tokenizer.cc b/tnet_io/KaldiLib/Tokenizer.cc new file mode 100644 index 0000000..0c49050 --- /dev/null +++ b/tnet_io/KaldiLib/Tokenizer.cc @@ -0,0 +1,53 @@ +#include "Tokenizer.h" +#include "string.h" + +namespace TNet +{ + //**************************************************************************** + //**************************************************************************** + void + Tokenizer:: + AddString(const char* pString) + { + // copy into string struct, which is more convenient + std::string aux_string(pString); + std::string aux_record; + std::string::size_type cur_pos = 0; + std::string::size_type old_pos = 0; + std::string::size_type search_start = 0; + + // make sure we have enough space + aux_record.reserve(aux_string.length()); + + // find all of separators and make a list of tokens + while(old_pos < std::string::npos) { + // find the next separator + cur_pos = aux_string.find_first_of(mSeparator, search_start); + + // if backslash is in front of separator, ignore this separator + if (cur_pos != 0 && cur_pos != std::string::npos && + pString[cur_pos - 1] == '\\') { + search_start = cur_pos + 1; + continue; + } + + // we don't want to have empty records + if (!(cur_pos == old_pos && mSkipEmpty)) { + // extract token + aux_record.insert(0, pString+old_pos, cur_pos==std::string::npos ? strlen(pString+old_pos) : cur_pos - old_pos); + // insert to list + this->push_back(aux_record); + + // we don't need the contents of the token + aux_record.erase(); + } + + // update old position so that it points behind the separator + old_pos = cur_pos < std::string::npos ? cur_pos + 1 : cur_pos; + search_start = old_pos; + } + } + + +} // namespace TNet + diff --git a/tnet_io/KaldiLib/Tokenizer.h b/tnet_io/KaldiLib/Tokenizer.h new file mode 100644 index 0000000..1be717b --- /dev/null +++ b/tnet_io/KaldiLib/Tokenizer.h @@ -0,0 +1,45 @@ +#include <list> +#include <string> + +namespace TNet { + /** + * @brief General string tokenizer + */ + class Tokenizer + : public std::list<std::string> + { + public: + // Constructors and Destructors ............................................ + Tokenizer(const char* pSeparator, bool skipEmpty = false) + : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty) + {} + + Tokenizer(const char* pString, const char* pSeparator, bool skipEmpty = false) + : std::list<std::string>(), mSeparator(pSeparator), mSkipEmpty(skipEmpty) + { AddString(pString); } + + ~Tokenizer() + {} + + /** + * @brief Parses a string and appends the tokens to the list + * @param pString string to parse + */ + void + AddString(const char* pString); + + /** + * @brief Constant accessor to the separators string + * @return Const refference + */ + const std::string& + Separator() const + {return mSeparator;} + + private: + std::string mSeparator; ///< holds the list of separators + bool mSkipEmpty; ///< if true, multiple separators will be regarded as one + }; // class Tokenizer +} // namespace TNet + + diff --git a/tnet_io/KaldiLib/Types.h b/tnet_io/KaldiLib/Types.h new file mode 100644 index 0000000..6a5bfac --- /dev/null +++ b/tnet_io/KaldiLib/Types.h @@ -0,0 +1,78 @@ +#ifndef TNet_Types_h +#define TNet_Types_h + +#ifdef HAVE_ATLAS +extern "C"{ + #include <cblas.h> + #include <clapack.h> +} +#endif + + +namespace TNet +{ + // TYPEDEFS .................................................................. +#if DOUBLEPRECISION + typedef double BaseFloat; +#else + typedef float BaseFloat; +#endif + +#ifndef UINT_16 + typedef unsigned short UINT_16 ; + typedef unsigned UINT_32 ; + typedef short INT_16 ; + typedef int INT_32 ; + typedef float FLOAT_32 ; + typedef double DOUBLE_64 ; +#endif + + + + // ........................................................................... + // The following declaration assumes that SSE instructions are enabled + // and that we are using GNU C/C++ compiler, which defines the __attribute__ + // notation. + // + // ENABLE_SSE is defined in <config.h>. Its value depends on options given + // in the configure phase of builidng the library +#if defined(__GNUC__ ) + // vector of four single floats + typedef float v4sf __attribute__((vector_size(16))); + // vector of two single doubles + typedef double v2sd __attribute__((vector_size(16))); + + typedef BaseFloat BaseFloat16Aligned __attribute__((aligned(16))) ; + + typedef union + { + v4sf v; + float f[4]; + } f4vector; + + typedef union + { + v2sd v; + double f[2]; + } d2vector; +#endif // ENABLE_SSE && defined(__GNUC__ ) + + + + typedef enum + { +#ifdef HAVE_ATLAS + TRANS = CblasTrans, + NO_TRANS = CblasNoTrans +#else + TRANS = 'T', + NO_TRANS = 'N' +#endif + } MatrixTrasposeType; + + + +} // namespace TNet + +#endif // #ifndef TNet_Types_h + diff --git a/tnet_io/KaldiLib/UserInterface.cc b/tnet_io/KaldiLib/UserInterface.cc new file mode 100644 index 0000000..b59a6c5 --- /dev/null +++ b/tnet_io/KaldiLib/UserInterface.cc @@ -0,0 +1,669 @@ +#include <stdexcept> +#include <sstream> +#include <stdarg.h> + +#include "UserInterface.h" +#include "StkStream.h" +#include "Features.h" + +namespace TNet +{ + //*************************************************************************** + //*************************************************************************** + int + npercents(const char *str) + { + int ret = 0; + while (*str) if (*str++ == '%') ret++; + return ret; + } + + + //*************************************************************************** + //*************************************************************************** + void + UserInterface:: + ReadConfig(const char *file_name) + { + std::string line_buf; + std::string::iterator chptr; + std::string key; + std::string value; + std::ostringstream ss; + int line_no = 0; + IStkStream i_stream; + + + i_stream.open(file_name, std::ios::binary); + if (!i_stream.good()) { + throw std::runtime_error(std::string("Cannot open input config file ") + + file_name); + } + i_stream >> std::ws; + + while (!i_stream.eof()) { + size_t i_pos; + + // read line + std::getline(i_stream, line_buf); + i_stream >> std::ws; + + if (i_stream.fail()) { + throw std::runtime_error(std::string("Error reading (") + + file_name + ":" + (ss << line_no,ss).str() + ")"); + } + + // increase line counter + line_no++; + + // cut comments + if (std::string::npos != (i_pos = line_buf.find('#'))) { + line_buf.erase(i_pos); + } + + // cut leading and trailing spaces + Trim(line_buf); + + // if empty line, then skip it + if (0 == line_buf.length()) { + continue; + } + + // line = line_buf.c_str(); + // chptr = parptr; + + chptr = line_buf.begin(); + + for (;;) { + // Replace speces by '_', which is removed in InsertConfigParam + while (isalnum(*chptr) || *chptr == '_' || *chptr == '-') { + chptr++; + } + + while (std::isspace(*chptr)) { + *chptr = '_'; + chptr++; + } + + if (*chptr != ':') { + break; + } + + chptr++; + + while (std::isspace(*chptr)) { + *chptr = '_'; + chptr++; + } + } + + if (*chptr != '=') { + throw std::runtime_error(std::string("Character '=' expected (") + + file_name + ":" + (ss.str(""),ss<<line_no,ss).str() + ")"); + } + + key.assign(line_buf.begin(), chptr); + + chptr++; + + value.assign(chptr, line_buf.end()); + + ParseHTKString(value, value); + InsertConfigParam(key.c_str(), value.c_str(), 'C'); + } + + i_stream.close(); + } + + + //*************************************************************************** + //*************************************************************************** + void + UserInterface:: + InsertConfigParam(const char *pParamName, const char *value, int optionChar) + { + std::string key(pParamName); + std::string::iterator i_key = key.begin(); + + while (i_key != key.end()) { + if (*i_key == '-' || *i_key == '_') { + i_key = key.erase(i_key); + } + else { + *i_key = toupper(*i_key); + i_key ++; + } + } + + mMap[key].mValue = value; + mMap[key].mRead = false; + mMap[key].mOption = optionChar; + } + + //*************************************************************************** + //*************************************************************************** + int + UserInterface:: + ParseOptions( + int argc, + char* argv[], + const char* pOptionMapping, + const char* pToolName) + { + int i; + int opt = '?'; + int optind; + bool option_must_follow = false; + char param[1024]; + char* value; + const char* optfmt; + const char* optarg; + char* chptr; + char* bptr; + char tstr[4] = " -?"; + unsigned long long option_mask = 0; + std::ostringstream ss; + + #define MARK_OPTION(ch) {if (isalpha(ch)) option_mask |= 1ULL << ((ch) - 'A');} + #define OPTION_MARK(ch) (isalpha(ch) && ((1ULL << ((ch) - 'A')) & option_mask)) + #define IS_OPTION(str) ((str)[0] == '-' && (isalpha((str)[1]) || (str)[1] == '-')) + + //search for the -A param + for (optind = 1; optind < argc; optind++) { + // we found "--", no -A + if (!strcmp(argv[optind], "--")) { + break; + } + + //repeat till we find -A + if (argv[optind][0] != '-' || argv[optind][1] != 'A') { + continue; + } + + // just "-A" form + if (argv[optind][2] != '\0') { + throw std::runtime_error(std::string("Unexpected argument '") + + (argv[optind] + 2) + "' after option '-A'"); + } + + for (i=0; i < argc; i++) { + // display all params + if(strchr(argv[i], ' ') || strchr(argv[i], '*')) + std::cout << '\'' << argv[i] << '\'' << " "; + else std::cout << argv[i] << " "; + } + + std::cout << std::endl; + + break; + } + + for (optind = 1; optind < argc; optind++) { + // find the '-C?' parameter (possible two configs) + if (!strcmp(argv[optind], "--")) break; + if (argv[optind][0] != '-' || argv[optind][1] != 'C') continue; + if (argv[optind][2] != '\0') { + ReadConfig(argv[optind] + 2); + } else if (optind+1 < argc && !IS_OPTION(argv[optind+1])) { + ReadConfig(argv[++optind]); + } else { + throw std::runtime_error("Config file name expected after option '-C'"); + } + } + + for (optind = 1; optind < argc; optind++) { + if (!strcmp(argv[optind], "--")) break; + if (argv[optind][0] != '-' || argv[optind][1] != '-') continue; + + bptr = new char[strlen(pToolName) + strlen(argv[optind]+2) + 2]; + strcat(strcat(strcpy(bptr, pToolName), ":"), argv[optind]+2); + value = strchr(bptr, '='); + if (!value) { + throw std::runtime_error(std::string("Character '=' expected after option '") + + argv[optind] + "'"); + } + + *value++ = '\0'; + + InsertConfigParam(bptr, value /*? value : "TRUE"*/, '-'); + delete [] bptr; + } + + for (optind = 1; optind < argc && IS_OPTION(argv[optind]); optind++) { + option_must_follow = false; + tstr[2] = opt = argv[optind][1]; + optarg = argv[optind][2] != '\0' ? argv[optind] + 2 : NULL; + + if (opt == '-' && !optarg) { // '--' terminates the option list + return optind+1; + } + if (opt == 'C' || opt == '-') { // C, A and long options have been already processed + if (!optarg) optind++; + continue; + } + if (opt == 'A') continue; + + chptr = strstr((char*)pOptionMapping, tstr); + if (chptr == NULL) { + throw std::runtime_error(std::string("Invalid command line option '-") + + static_cast<char>(opt) + "'"); + } + + chptr += 3; + while (std::isspace(*chptr)) { + chptr++; + } + + if (!chptr || chptr[0] == '-') {// Option without format string will be ignored + optfmt = " "; + } else { + optfmt = chptr; + while (*chptr && !std::isspace(*chptr)) { + chptr++; + } + if (!*chptr) { + throw std::runtime_error("Fatal: Unexpected end of optionMap string"); + } + } + for (i = 0; !std::isspace(*optfmt); optfmt++) { + while (std::isspace(*chptr)) chptr++; + value = chptr; + while (*chptr && !std::isspace(*chptr)) chptr++; + assert(static_cast<unsigned int>(chptr-value+1) < sizeof(param)); + strncat(strcat(strcpy(param, pToolName), ":"), value, chptr-value); + param[chptr-value+strlen(pToolName)+1] = '\0'; + switch (*optfmt) { + case 'n': + value = strchr(param, '='); + if (value) *value = '\0'; + InsertConfigParam(param, + value ? value + 1: "TRUE", opt); + break; + + case 'l': + case 'o': + case 'r': + i++; + if (!optarg && (optind+1==argc || IS_OPTION(argv[optind+1]))) { + if (*optfmt == 'r' || *optfmt == 'l') { + throw std::runtime_error(std::string("Argument ") + + (ss<<i,ss).str() + " of option '-" + + static_cast<char>(opt) + "' expected"); + } + optfmt = " "; // Stop reading option arguments + break; + } + if (!optarg) optarg = argv[++optind]; + if (*optfmt == 'o') { + option_must_follow = (bool) 1; + } + bptr = NULL; + + // For repeated use of option with 'l' (list) format, append + // ',' and argument string to existing config parameter value. + if (*optfmt == 'l' && OPTION_MARK(opt)) { + bptr = strdup(GetStr(param, "")); + if (bptr == NULL) throw std::runtime_error("Insufficient memory"); + bptr = (char*) realloc(bptr, strlen(bptr) + strlen(optarg) + 2); + if (bptr == NULL) throw std::runtime_error("Insufficient memory"); + strcat(strcat(bptr, ","), optarg); + optarg = bptr; + } + MARK_OPTION(opt); + InsertConfigParam(param, optarg, opt); + free(bptr); + optarg = NULL; + break; + + default : + throw std::runtime_error(std::string("Fatal: Invalid character '") + + *optfmt + "' in optionMap after " + tstr); + } + } + if (optarg) { + throw std::runtime_error(std::string("Unexpected argument '") + + optarg + "' after option '-" + + static_cast<char>(opt) + "'"); + } + } + + for (i = optind; i < argc && !IS_OPTION(argv[i]); i++) + {} + + if (i < argc) { + throw std::runtime_error(std::string("No option expected after first non-option argument '") + + argv[optind] + "'"); + } + + if (option_must_follow) { + throw std::runtime_error(std::string("Option '-") + + static_cast<char>(opt) + + "' with optional argument must not be the last option"); + } + + return optind; + } + + + //*************************************************************************** + //*************************************************************************** + int + UserInterface:: + GetFeatureParams( + int * derivOrder, + int ** derivWinLens, + int * startFrmExt, + int * endFrmExt, + char ** CMNPath, + char ** CMNFile, + const char ** CMNMask, + char ** CVNPath, + char ** CVNFile, + const char ** CVNMask, + const char ** CVGFile, + const char * pToolName, + int pseudoModeule) + { + const char * str; + int targetKind; + char * chrptr; + char paramName[32]; + const char * CMNDir; + const char * CVNDir; + + strcpy(paramName, pToolName); + strcat(paramName, pseudoModeule == 1 ? "SPARM1:" : + pseudoModeule == 2 ? "SPARM2:" : ""); + + chrptr = paramName + strlen(paramName); + + strcpy(chrptr, "STARTFRMEXT"); + *startFrmExt = GetInt(paramName, 0); + strcpy(chrptr, "ENDFRMEXT"); + *endFrmExt = GetInt(paramName, 0); + + *CMNPath = *CVNPath = NULL; + strcpy(chrptr, "CMEANDIR"); + CMNDir = GetStr(paramName, NULL); + strcpy(chrptr, "CMEANMASK"); + *CMNMask = GetStr(paramName, NULL); + + if (*CMNMask != NULL) { + *CMNPath = (char*) malloc((CMNDir ? strlen(CMNDir) : 0) + npercents(*CMNMask) + 2); + if (*CMNPath == NULL) throw std::runtime_error("Insufficient memory"); + if (CMNDir != NULL) strcat(strcpy(*CMNPath, CMNDir), "/"); + *CMNFile = *CMNPath + strlen(*CMNPath); + } + strcpy(chrptr, "VARSCALEDIR"); + CVNDir = GetStr(paramName, NULL); + strcpy(chrptr, "VARSCALEMASK"); + *CVNMask = GetStr(paramName, NULL); + + + if (*CVNMask != NULL) { + *CVNPath = (char*) malloc((CVNDir ? strlen(CVNDir) : 0) + npercents(*CVNMask) + 2); + if (*CVNPath == NULL) throw std::runtime_error("Insufficient memory"); + if (CVNDir != NULL) strcat(strcpy(*CVNPath, CVNDir), "/"); + *CVNFile = *CVNPath + strlen(*CVNPath); + } + strcpy(chrptr, "VARSCALEFN"); + *CVGFile = GetStr(paramName, NULL); + strcpy(chrptr, "TARGETKIND"); + str = GetStr(paramName, "ANON"); + + targetKind = FeatureRepository::ReadParmKind(str, false); + + if (targetKind == -1) { + throw std::runtime_error(std::string("Invalid TARGETKIND = '") + + str + "'"); + } + + strcpy(chrptr, "DERIVWINDOWS"); + if ((str = GetStr(paramName, NULL)) != NULL) { + long lval; + *derivOrder = 0; + *derivWinLens = NULL; + + if (NULL != str) + { + while ((str = strtok((char *) str, " \t_")) != NULL) + { + lval = strtol(str, &chrptr, 0); + if (!*str || *chrptr) { + throw std::runtime_error("Integers separated by '_' expected for parameter DERIVWINDOWS"); + } + *derivWinLens = (int *)realloc(*derivWinLens, ++*derivOrder*sizeof(int)); + if (*derivWinLens == NULL) throw std::runtime_error("Insufficient memory"); + (*derivWinLens)[*derivOrder-1] = lval; + str = NULL; + } + } + + return targetKind; + } + *derivOrder = targetKind & PARAMKIND_T ? 3 : + targetKind & PARAMKIND_A ? 2 : + targetKind & PARAMKIND_D ? 1 : 0; + + if (*derivOrder || targetKind != PARAMKIND_ANON) { + *derivWinLens = (int *) malloc(3 * sizeof(int)); + if (*derivWinLens == NULL) throw std::runtime_error("Insufficient memory"); + + strcpy(chrptr, "DELTAWINDOW"); + (*derivWinLens)[0] = GetInt(paramName, 2); + strcpy(chrptr, "ACCWINDOW"); + (*derivWinLens)[1] = GetInt(paramName, 2); + strcpy(chrptr, "THIRDWINDOW"); + (*derivWinLens)[2] = GetInt(paramName, 2); + return targetKind; + } + *derivWinLens = NULL; + *derivOrder = -1; + return targetKind; + } + + + //*************************************************************************** + //*************************************************************************** + UserInterface::ValueRecord* + UserInterface:: + GetParam(const char* pParamName) + { + MapType::iterator it; + + // this is done only for convenience. in the loop we will increase the + // pointer again + pParamName--; + + // we iteratively try to find the param name in the map. if an attempt + // fails, we strip off all characters until the first ':' and we search + // again + do { + pParamName++; + it = mMap.find(pParamName); + } while ((it == mMap.end()) && (NULL != (pParamName = strchr(pParamName, ':')))); + + if (it == mMap.end()) { + return NULL; + } + else { + it->second.mRead = true; + return &(it->second); + } + } + + + //*************************************************************************** + //*************************************************************************** + const char * + UserInterface:: + GetStr( + const char * pParamName, + const char * default_value) + { + ValueRecord* p_val = GetParam(pParamName); + + if (NULL == p_val) { + return default_value; + } + else { + return p_val->mValue.c_str(); + } + } + + + //*************************************************************************** + //*************************************************************************** + long + UserInterface:: + GetInt( + const char *pParamName, + long default_value) + { + char *chrptr; + ValueRecord* p_val = GetParam(pParamName); + + if (NULL == p_val) { + return default_value; + } + + const char *val = p_val->mValue.c_str(); + default_value = strtol(val, &chrptr, 0); + if (!*val || *chrptr) { + throw std::runtime_error(std::string("Integer number expected for ") + + pParamName + " but found '" + val + "'"); + } + return default_value; + } + + //*************************************************************************** + //*************************************************************************** + float + UserInterface:: + GetFlt( + const char * pParamName, + float default_value) + { + char *chrptr; + ValueRecord* p_val = GetParam(pParamName); + + if (NULL == p_val) { + return default_value; + } + + const char *val = p_val->mValue.c_str(); + default_value = strtod(val, &chrptr); + if (!*val || *chrptr) { + throw std::runtime_error(std::string("Decimal number expected for ") + + pParamName + " but found '" + val + "'"); + } + return default_value; + } + + //*************************************************************************** + //*************************************************************************** + bool + UserInterface:: + GetBool( + const char * pParamName, + bool default_value) + { + ValueRecord* p_val = GetParam(pParamName); + + if (NULL == p_val) { + return default_value; + } + + const char* val = p_val->mValue.c_str(); + + if (!strcasecmp(val, "TRUE") || !strcmp(val, "T")) return 1; + if (strcasecmp(val, "FALSE") && strcmp(val, "F")) { + throw std::runtime_error(std::string("TRUE or FALSE expected for ") + + pParamName + " but found '" + val + "'"); + } + return false; + } + + //*************************************************************************** + //*************************************************************************** + // '...' are pairs: string and corresponding integer value , terminated by NULL + int + UserInterface:: + GetEnum( + const char * pParamName, + int default_value, + ...) + { + ValueRecord* p_val = GetParam(pParamName); + + if (NULL == p_val) { + return default_value; + } + + const char* val = p_val->mValue.c_str(); + char* s; + int i = 0, cnt = 0, l = 0; + va_list ap; + + va_start(ap, default_value); + while ((s = va_arg(ap, char *)) != NULL) { + l += strlen(s) + 2; + ++cnt; + i = va_arg(ap, int); + if (!strcmp(val, s)) break; + } + va_end(ap); + + if (s) { + return i; + } + + //To report error, create string listing all possible values + s = (char*) malloc(l + 1); + s[0] = '\0'; + va_start(ap, default_value); + for (i = 0; i < cnt; i++) { + strcat(s, va_arg(ap, char *)); + va_arg(ap, int); + if (i < cnt - 2) strcat(s, ", "); + else if (i == cnt - 2) strcat(s, " or "); + } + + va_end(ap); + + throw std::runtime_error(std::string(s) + " expected for " + + pParamName + " but found '" + val + "'"); + + return 0; + } + + + //*************************************************************************** + //*************************************************************************** + void + UserInterface:: + PrintConfig(std::ostream& rStream) + { + rStream << "Configuration Parameters[" << mMap.size() << "]\n"; + for (MapType::iterator it = mMap.begin(); it != mMap.end(); ++it) { + rStream << (it->second.mRead ? " " : "# ") + << std::setw(35) << std::left << it->first << " = " + << std::setw(30) << std::left << it->second.mValue + << " # -" << it->second.mOption << std::endl; + } + } + + //*************************************************************************** + //*************************************************************************** + void + UserInterface:: + CheckCommandLineParamUse() + { + for (MapType::iterator it = mMap.begin(); it != mMap.end(); ++it) { + if (!it->second.mRead && it->second.mOption != 'C') { + Error("Unexpected command line parameter " + it->first); + } + } + } + +} diff --git a/tnet_io/KaldiLib/UserInterface.h b/tnet_io/KaldiLib/UserInterface.h new file mode 100644 index 0000000..fa189e7 --- /dev/null +++ b/tnet_io/KaldiLib/UserInterface.h @@ -0,0 +1,166 @@ +#ifndef TNet_UserInterface_h +#define TNet_UserInterface_h + +#include <iostream> +#include <cstdlib> +#include <string> +#include <map> + +namespace TNet +{ + /** ************************************************************************** + ** ************************************************************************** + */ + class UserInterface + { + public: + struct ValueRecord { + std::string mValue; + char mOption; + bool mRead; + }; + + + void InsertConfigParam( + const char *param_name, + const char *value, + int optionChar); + + + void + ReadConfig(const char *pFileName); + + + void + CheckCommandLineParamUse(); + + + /** + * @brief Retreives the content of a parameter + * @param pParamName Name of the parameter to look for + * @return Returns the pointer to the ValueRecord structure if success, + * otherwise return NULL + * + * We iteratively try to find the param name in the map. If an attempt + * fails, we strip off all characters until the first occurance of ':' + * and we search again + */ + ValueRecord* + GetParam(const char* pParamName); + + + /** + * @brief Returns the parameter's value as string + * + * @param param_name Parameter name + * @param default_value Value, which is returned in case the parameter + * was not found + * + * @return Pointer to the begining of the string if success, default_value + * otherwise + */ + const char* + GetStr( const char *param_name, const char *default_value); + + + /** + * @brief Returns the parameter's value as int + * + * @param param_name Parameter name + * @param default_value Value, which is returned in case the parameter + * was not found + * + * @return Returns the integer value if success, default_value + * otherwise + */ + long + GetInt( const char *param_name, long default_value); + + + /** + * @brief Returns the parameter's value as float + * + * @param param_name Parameter name + * @param default_value Value, which is returned in case the parameter + * was not found + * + * @return Returns the float value if success, default_value + * otherwise + */ + float + GetFlt( const char *param_name, float default_value); + + + /** + * @brief Returns the parameter's value as bool + * + * @param param_name Parameter name + * @param default_value Value, which is returned in case the parameter + * was not found + * + * @return Returns the bool value if success, default_value + * otherwise + * + * Note that true is returned if the value is 'TRUE' or 'T', false is + * returned if the value is 'FALSE' or 'F'. Otherwise exception is thrown + */ + bool + GetBool(const char *param_name, bool default_value); + + + /** + * @brief Returns the parameter's value as enum integer + * + * @param param_name Parameter name + * @param default_value Value, which is returned in case the parameter + * was not found + * + * @return Returns the index value if success, default_value + * otherwise + * + * Variable arguments specify the possible values of this parameter. If the + * value does not match any of these, exception is thrown. + */ + int + GetEnum( const char *param_name, int default_value, ...); + + + int GetFeatureParams( + int *derivOrder, + int **derivWinLens, + int *startFrmExt, + int *endFrmExt, + char **CMNPath, + char **CMNFile, + const char **CMNMask, + char **CVNPath, + char **CVNFile, + const char **CVNMask, + const char **CVGFile, + const char *toolName, + int pseudoModeule); + + + int ParseOptions( + int argc, + char* argv[], + const char* optionMapping, + const char* toolName); + + + /** + * @brief Send the defined paramaters to a stream + * + * @param rStream stream to use + */ + void + PrintConfig(std::ostream& rStream); + + public: + typedef std::map<std::string, ValueRecord> MapType; + MapType mMap; + }; +} + +#endif + diff --git a/tnet_io/KaldiLib/Vector.cc b/tnet_io/KaldiLib/Vector.cc new file mode 100644 index 0000000..020bae2 --- /dev/null +++ b/tnet_io/KaldiLib/Vector.cc @@ -0,0 +1,110 @@ +#ifndef TNet_Vector_cc +#define TNet_Vector_cc + +#include <cstdlib> +#include <cmath> +#include <cstring> +#include <fstream> +#include <iomanip> +#include "Common.h" + +#ifdef HAVE_ATLAS +extern "C"{ + #include <cblas.h> +} +#endif + +#include "Common.h" +#include "Matrix.h" +#include "Vector.h" + +namespace TNet +{ + +#ifdef HAVE_ATLAS + template<> + float + BlasDot<>(const Vector<float>& rA, const Vector<float>& rB) + { + assert(rA.mDim == rB.mDim); + return cblas_sdot(rA.mDim, rA.pData(), 1, rB.pData(), 1); + } + + template<> + double + BlasDot<>(const Vector<double>& rA, const Vector<double>& rB) + { + assert(rA.mDim == rB.mDim); + return cblas_ddot(rA.mDim, rA.pData(), 1, rB.pData(), 1); + } + + template<> + Vector<float>& + Vector<float>:: + BlasAxpy(const float alpha, const Vector<float>& rV) + { + assert(mDim == rV.mDim); + cblas_saxpy(mDim, alpha, rV.pData(), 1, mpData, 1); + return *this; + } + + template<> + Vector<double>& + Vector<double>:: + BlasAxpy(const double alpha, const Vector<double>& rV) + { + assert(mDim == rV.mDim); + cblas_daxpy(mDim, alpha, rV.pData(), 1, mpData, 1); + return *this; + } + + template<> + Vector<int>& + Vector<int>:: + BlasAxpy(const int alpha, const Vector<int>& rV) + { + assert(mDim == rV.mDim); + for(int i=0; i<Dim(); i++) { + (*this)[i] += rV[i]; + } + return *this; + } + + + template<> + Vector<float>& + Vector<float>:: + BlasGemv(const float alpha, const Matrix<float>& rM, MatrixTrasposeType trans, const Vector<float>& rV, const float beta) + { + assert((trans == NO_TRANS && rM.Cols() == rV.mDim && rM.Rows() == mDim) + || (trans == TRANS && rM.Rows() == rV.mDim && rM.Cols() == mDim)); + + cblas_sgemv(CblasRowMajor, static_cast<CBLAS_TRANSPOSE>(trans), rM.Rows(), rM.Cols(), alpha, rM.pData(), rM.Stride(), + rV.pData(), 1, beta, mpData, 1); + return *this; + } + + + + template<> + Vector<double>& + Vector<double>:: + BlasGemv(const double alpha, const Matrix<double>& rM, MatrixTrasposeType trans, const Vector<double>& rV, const double beta) + { + assert((trans == NO_TRANS && rM.Cols() == rV.mDim && rM.Rows() == mDim) + || (trans == TRANS && rM.Rows() == rV.mDim && rM.Cols() == mDim)); + + cblas_dgemv(CblasRowMajor, static_cast<CBLAS_TRANSPOSE>(trans), rM.Rows(), rM.Cols(), alpha, rM.pData(), rM.Stride(), + rV.pData(), 1, beta, mpData, 1); + return *this; + } + + +#else + #error Routines in this section are not implemented yet without BLAS +#endif + +} // namespace TNet + + +#endif // TNet_Vector_tcc diff --git a/tnet_io/KaldiLib/Vector.h b/tnet_io/KaldiLib/Vector.h new file mode 100644 index 0000000..384c5d2 --- /dev/null +++ b/tnet_io/KaldiLib/Vector.h @@ -0,0 +1,496 @@ +// +// C++ Interface: %{MODULE} +// +// Description: +// +// +// Author: %{AUTHOR} <%{EMAIL}>, (C) %{YEAR} +// +// Copyright: See COPYING file that comes with this distribution +// +// + +#ifndef TNet_Vector_h +#define TNet_Vector_h + +#include <cstddef> +#include <cstdlib> +#include <stdexcept> +#include <iostream> + +#ifdef HAVE_ATLAS +extern "C"{ + #include <cblas.h> + #include <clapack.h> +} +#endif + +#include "Common.h" +#include "MathAux.h" +#include "Types.h" +#include "Error.h" + +namespace TNet +{ + template<typename _ElemT> class Vector; + template<typename _ElemT> class SubVector; + template<typename _ElemT> class Matrix; + template<typename _ElemT> class SpMatrix; + + // we need to declare the friend functions here + template<typename _ElemT> + std::ostream & operator << (std::ostream & rOut, const Vector<_ElemT> & rV); + + template<typename _ElemT> + std::istream & operator >> (std::istream & rIn, Vector<_ElemT> & rV); + + template<typename _ElemT> + _ElemT + BlasDot(const Vector<_ElemT>& rA, const Vector<_ElemT>& rB); + + /** ************************************************************************** + ** ************************************************************************** + * @brief Provides a matrix abstraction class + * + * This class provides a way to work with matrices in TNet. + * It encapsulates basic operations and memory optimizations. + * + */ + template<typename _ElemT> + class Vector + { + public: + + /// defines a type of this + typedef Vector<_ElemT> ThisType; + + + Vector(): mpData(NULL) +#ifdef STK_MEMALIGN_MANUAL + ,mpFreeData(NULL) +#endif + , mDim(0) + {} + + /** + * @brief Copy constructor + * @param rV + */ + Vector(const Vector<_ElemT>& rV) + { mpData=NULL; Init(rV.Dim()); Copy(rV); } + + + /* Type conversion constructor. */ + template<typename _ElemU> + explicit Vector(const Vector<_ElemU>& rV) + { mpData=NULL; Init(rV.Dim()); Copy(rV); } + + + Vector(const _ElemT* ppData, const size_t s) + { mpData=NULL; Init(s); Copy(ppData); } + + explicit Vector(const size_t s, bool clear=true) + { mpData=NULL; Init(s,clear); } + + ~Vector() + { Destroy(); } + + Vector<_ElemT> &operator = (const Vector <_ElemT> &other) + { Init(other.Dim()); Copy(other); return *this; } // Needed for inclusion in std::vector + + Vector<_ElemT>& + Init(size_t length, bool clear=true); + + /** + * @brief Dealocates the window from memory and resets the dimensions to (0) + */ + void + Destroy(); + + /** + * @brief Returns @c true if vector is initialized + */ + bool + IsInitialized() const + { return mpData != NULL; } + + /** + * @brief Sets all elements to 0 + */ + void + Zero(); + + void + Set(_ElemT f); + + inline size_t + Dim() const + { return mDim; } + + /** + * @brief Returns size of matrix in memory (in bytes) + */ + inline size_t + MSize() const + { + return (mDim + (((16 / sizeof(_ElemT)) - mDim%(16 / sizeof(_ElemT))) + % (16 / sizeof(_ElemT)))) * sizeof(_ElemT); + } + + /** + * @brief Gives access to the vector memory area + * @return pointer to the first field + */ + inline _ElemT* + pData() + { return mpData; } + + /** + * @brief Gives access to the vector memory area + * @return pointer to the first field (const version) + */ + inline const _ElemT* + pData() const + { return mpData; } + + /** + * @brief Gives access to a specified vector element (const). + */ + inline _ElemT + operator [] (size_t i) const + { +#ifdef PARANOID + assert(i<mDim); +#endif + return *(mpData + i); + } + + /** + * @brief Gives access to a specified vector element (non-const). + */ + inline _ElemT & + operator [] (size_t i) + { +#ifdef PARANOID + assert(i<mDim); +#endif + return *(mpData + i); + } + + /** + * @brief Gives access to a specified vector element (const). + */ + inline _ElemT + operator () (size_t i) const + { +#ifdef PARANOID + assert(i<mDim); +#endif + return *(mpData + i); + } + + /** + * @brief Gives access to a specified vector element (non-const). + */ + inline _ElemT & + operator () (size_t i) + { +#ifdef PARANOID + assert(i<mDim); +#endif + return *(mpData + i); + } + + /** + * @brief Returns a matrix sub-range + * @param o Origin + * @param l Length + * See @c SubVector class for details + */ + SubVector<_ElemT> + Range(const size_t o, const size_t l) + { return SubVector<_ElemT>(*this, o, l); } + + /** + * @brief Returns a matrix sub-range + * @param o Origin + * @param l Length + * See @c SubVector class for details + */ + const SubVector<_ElemT> + Range(const size_t o, const size_t l) const + { return SubVector<_ElemT>(*this, o, l); } + + + + //######################################################################## + //######################################################################## + + /// Copy data from another vector + Vector<_ElemT>& + Copy(const Vector<_ElemT>& rV); + + /// Copy data from another vector of a different type. + template<typename _ElemU> Vector<_ElemT>& + Copy(const Vector<_ElemU>& rV); + + + /// Load data into the vector + Vector<_ElemT>& + Copy(const _ElemT* ppData); + + Vector<_ElemT>& + CopyVectorizedMatrixRows(const Matrix<_ElemT> &rM); + + Vector<_ElemT>& + RemoveElement(size_t i); + + Vector<_ElemT>& + ApplyLog(); + + Vector<_ElemT>& + ApplyLog(const Vector<_ElemT>& rV);//ApplyLog to rV and put the result in (*this) + + Vector<_ElemT>& + ApplyExp(); + + Vector<_ElemT>& + ApplySoftMax(); + + Vector<_ElemT>& + Invert(); + + Vector<_ElemT>& + DotMul(const Vector<_ElemT>& rV); // Multiplies each element (*this)(i) by rV(i). + + Vector<_ElemT>& + BlasAxpy(const _ElemT alpha, const Vector<_ElemT>& rV); + + Vector<_ElemT>& + BlasGemv(const _ElemT alpha, const Matrix<_ElemT>& rM, const MatrixTrasposeType trans, const Vector<_ElemT>& rV, const _ElemT beta = 0.0); + + + //######################################################################## + //######################################################################## + + Vector<_ElemT>& + Add(const Vector<_ElemT>& rV) + { return BlasAxpy(1.0, rV); } + + Vector<_ElemT>& + Subtract(const Vector<_ElemT>& rV) + { return BlasAxpy(-1.0, rV); } + + Vector<_ElemT>& + AddScaled(_ElemT alpha, const Vector<_ElemT>& rV) + { return BlasAxpy(alpha, rV); } + + Vector<_ElemT>& + Add(_ElemT c); + + Vector<_ElemT>& + MultiplyElements(const Vector<_ElemT>& rV); + + // @brief elementwise : rV.*rR+beta*this --> this + Vector<_ElemT>& + MultiplyElements(_ElemT alpha, const Vector<_ElemT>& rV, const Vector<_ElemT>& rR,_ElemT beta); + + Vector<_ElemT>& + DivideElements(const Vector<_ElemT>& rV); + + /// @brief elementwise : rV./rR+beta*this --> this + Vector<_ElemT>& + DivideElements(_ElemT alpha, const Vector<_ElemT>& rV, const Vector<_ElemT>& rR,_ElemT beta); + + Vector<_ElemT>& + Subtract(_ElemT c); + + Vector<_ElemT>& + Scale(_ElemT c); + + + //######################################################################## + //######################################################################## + + /// Performs a row stack of the matrix rMa + Vector<_ElemT>& + MatrixRowStack(const Matrix<_ElemT>& rMa); + + // Extracts a row of the matrix rMa. .. could also do this with vector.Copy(rMa[row]). + Vector<_ElemT>& + Row(const Matrix<_ElemT>& rMa, size_t row); + + // Extracts a column of the matrix rMa. + Vector<_ElemT>& + Col(const Matrix<_ElemT>& rMa, size_t col); + + // Takes all elements to a power. + Vector<_ElemT>& + Power(_ElemT power); + + _ElemT + Max() const; + + _ElemT + Min() const; + + /// Returns sum of the elements + _ElemT + Sum() const; + + /// Returns sum of the elements + Vector<_ElemT>& + AddRowSum(const Matrix<_ElemT>& rM); + + /// Returns sum of the elements + Vector<_ElemT>& + AddColSum(const Matrix<_ElemT>& rM); + + /// Returns log(sum(exp())) without exp overflow + _ElemT + LogSumExp() const; + + //######################################################################## + //######################################################################## + + friend std::ostream & + operator << <> (std::ostream& rOut, const Vector<_ElemT>& rV); + + friend _ElemT + BlasDot<>(const Vector<_ElemT>& rA, const Vector<_ElemT>& rB); + + /** + * Computes v1^T * M * v2. + * Not as efficient as it could be where v1==v2 (but no suitable blas + * routines available). + */ + _ElemT + InnerProduct(const Vector<_ElemT> &v1, const Matrix<_ElemT> &M, const Vector<_ElemT> &v2) const; + + + //########################################################################## + //########################################################################## + //protected: + public: + /// data memory area + _ElemT* mpData; +#ifdef STK_MEMALIGN_MANUAL + /// data to be freed (in case of manual memalignment use, see common.h) + _ElemT* mpFreeData; +#endif + size_t mDim; ///< Number of elements + }; // class Vector + + + + + /** + * @brief Represents a non-allocating general vector which can be defined + * as a sub-vector of higher-level vector + */ + template<typename _ElemT> + class SubVector : public Vector<_ElemT> + { + protected: + /// Constructor + SubVector(const Vector<_ElemT>& rT, + const size_t origin, + const size_t length) + { + assert(origin+length <= rT.mDim); + Vector<_ElemT>::mpData = rT.mpData+origin; + Vector<_ElemT>::mDim = length; + } + //only Vector class can call this protected constructor + friend class Vector<_ElemT>; + + public: + /// Constructor + SubVector(Vector<_ElemT>& rT, + const size_t origin, + const size_t length) + { + assert(origin+length <= rT.mDim); + Vector<_ElemT>::mpData = rT.mpData+origin; + Vector<_ElemT>::mDim = length; + } + + + /** + * @brief Constructs a vector representation out of a standard array + * + * @param pData pointer to data array to associate with this vector + * @param length length of this vector + */ + inline + SubVector(_ElemT *ppData, + size_t length) + { + Vector<_ElemT>::mpData = ppData; + Vector<_ElemT>::mDim = length; + } + + + /** + * @brief Destructor + */ + ~SubVector() + { + Vector<_ElemT>::mpData = NULL; + } + }; + + + // Useful shortcuts + typedef Vector<BaseFloat> BfVector; + typedef SubVector<BaseFloat> BfSubVector; + + //Adding two vectors of different types + template <typename _ElemT, typename _ElemU> + void Add(Vector<_ElemT>& rDst, const Vector<_ElemU>& rSrc) + { + assert(rDst.Dim() == rSrc.Dim()); + const _ElemU* p_src = rSrc.pData(); + _ElemT* p_dst = rDst.pData(); + + for(size_t i=0; i<rSrc.Dim(); i++) { + *p_dst++ += (_ElemT)*p_src++; + } + } + + + //Scales adding two vectors of different types + template <typename _ElemT, typename _ElemU> + void AddScaled(Vector<_ElemT>& rDst, const Vector<_ElemU>& rSrc, _ElemT scale) + { + assert(rDst.Dim() == rSrc.Dim()); + + Vector<_ElemT> tmp(rSrc); + rDst.BlasAxpy(scale, tmp); + +/* + const _ElemU* p_src = rSrc.pData(); + _ElemT* p_dst = rDst.pData(); + + for(size_t i=0; i<rDst.Dim(); i++) { + *p_dst++ += *p_src++ * scale; + } +*/ + } + + +} // namespace TNet + +//***************************************************************************** +//***************************************************************************** +// we need to include the implementation +#include "Vector.tcc" + +/****************************************************************************** + ****************************************************************************** + * The following section contains specialized template definitions + * whose implementation is in Vector.cc + */ + + +#endif // #ifndef TNet_Vector_h diff --git a/tnet_io/KaldiLib/Vector.tcc b/tnet_io/KaldiLib/Vector.tcc new file mode 100644 index 0000000..751ffa7 --- /dev/null +++ b/tnet_io/KaldiLib/Vector.tcc @@ -0,0 +1,638 @@ +/** @file Vector.tcc + * This is an internal header file, included by other library headers. + * You should not attempt to use it directly. + */ + +#ifndef TNet_Vector_tcc +#define TNet_Vector_tcc + +#include <cstdlib> +#include <cmath> +#include <cstring> +#include <fstream> +#include <iomanip> +#include "Common.h" + +#ifdef HAVE_ATLAS +extern "C"{ + #include <cblas.h> +} +#endif + +#include "Common.h" +#include "MathAux.h" +#include "Matrix.h" + +namespace TNet +{ + //****************************************************************************** + //****************************************************************************** + template<typename _ElemT> + inline Vector<_ElemT>& + Vector<_ElemT>:: + Init(const size_t length, bool clear) + { + if(mpData != NULL) Destroy(); + if(length==0){ + mpData=NULL; +#ifdef STK_MEMALIGN_MANUAL + mpFreeData=NULL; +#endif + mDim=0; + return *this; + } + size_t size; + void* data; + void* free_data; + + size = align<16>(length * sizeof(_ElemT)); + + if (NULL != (data = stk_memalign(16, size, &free_data))) { + mpData = static_cast<_ElemT*> (data); +#ifdef STK_MEMALIGN_MANUAL + mpFreeData = static_cast<_ElemT*> (free_data); +#endif + mDim = length; + } else { + throw std::bad_alloc(); + } + if(clear) Zero(); + return *this; + } + + + //****************************************************************************** + //****************************************************************************** + /// Copy data from another vector + template<typename _ElemT> + inline Vector<_ElemT>& + Vector<_ElemT>:: + Copy(const Vector<_ElemT>& rV) { + assert(Dim() == rV.Dim()); + Copy(rV.mpData); + return *this; + } + + /// Load data into the vector + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + Copy(const _ElemT* ppData) { + std::memcpy(this->mpData, ppData, Dim() * sizeof(_ElemT)); + return *this; + } + + template<typename _ElemT> + template<typename _ElemU> + Vector<_ElemT>& + Vector<_ElemT>:: + Copy(const Vector<_ElemU> &other){ + assert(Dim()==other.Dim()); + size_t D=Dim(); + for(size_t d=0;d<D;d++) (*this)(d) = (_ElemT) other[d]; + return *this; + } + + + //****************************************************************************** + //****************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + CopyVectorizedMatrixRows(const Matrix<_ElemT> &rM) { +// TraceLog("Dim = "+to_string(Dim())+", Rows = "+to_string(rM.Rows())+", Cols = "+to_string(rM.Cols())); + assert(Dim() == rM.Cols()*rM.Rows()); + size_t nCols = rM.Cols(); + for(size_t r=0; r<rM.Rows(); r++) + Range(r*nCols, nCols).Copy(rM[r]); + return *this; + } + + + //**************************************************************************** + //**************************************************************************** + // Remove element from the vector. The vector is non reallocated + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + RemoveElement(size_t i) { + assert(i < mDim && "Access out of vector"); + for(size_t j = i + 1; j < mDim; j++) + this->mpData[j - 1] = this->mpData[j]; + mDim--; + return *this; + } + + //**************************************************************************** + //**************************************************************************** + // The destructor + template<typename _ElemT> + inline void + Vector<_ElemT>:: + Destroy() + { + // we need to free the data block if it was defined +#ifndef STK_MEMALIGN_MANUAL + if (NULL != mpData) free(mpData); +#else + if (NULL != mpData) free(mpFreeData); + mpFreeData = NULL; +#endif + + mpData = NULL; + mDim = 0; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + inline void + Vector<_ElemT>:: + Zero() + { + std::memset(mpData, 0, mDim * sizeof(_ElemT)); + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + inline void + Vector<_ElemT>:: + Set(_ElemT f) + { + for(size_t i=0;i<mDim;i++) mpData[i] = f; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + MatrixRowStack(const Matrix<_ElemT>& rMa) + { + assert(mDim == rMa.Cols() * rMa.Rows()); + + _ElemT* inc_data = mpData; + const size_t cols = rMa.Cols(); + + for (size_t i = 0; i < rMa.Rows(); i++) + { + // copy the data to the propper position + memcpy(inc_data, rMa[i], cols * sizeof(_ElemT)); + + // set new copy position + inc_data += cols; + } + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + Row(const Matrix<_ElemT> &rMa, size_t row) + { + assert(row < rMa.Rows()); + const _ElemT *mRow = rMa.pRowData(row); + // if(mDim != rMa.Cols()) Init(rMa.Cols()); // automatically resize. + memcpy(mpData, mRow, sizeof(_ElemT)*mDim); + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + Power(_ElemT power) // takes elements to a power. Throws exception if could not. + { + for(size_t i=0;i<Dim();i++){ + _ElemT tmp = (*this)(i); + (*this)(i) = pow(tmp, power); + if((*this)(i) == HUGE_VAL) + throw std::runtime_error((std::string)"Error in Vector::power, could not take " +to_string(tmp)+ " to power " +to_string((*this)(i))); + } + return (*this); + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + _ElemT + Vector<_ElemT>:: + Max() const + { + if(Dim()==0) throw std::runtime_error("Error in Vector::Max(), empty vector\n"); + _ElemT ans = (*this)(0); + for(size_t i=1;i<Dim();i++) ans = std::max(ans, (*this)(i)); + return ans; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + _ElemT + Vector<_ElemT>:: + Min() const + { + if(Dim()==0) throw std::runtime_error("Error in Vector::Min(), empty vector\n"); + _ElemT ans = (*this)(0); + for(size_t i=1;i<Dim();i++) ans = std::min(ans, (*this)(i)); + return ans; + } + + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + Col(const Matrix<_ElemT> &rMa, size_t col) + { + assert(col < rMa.Cols()); + // if(mDim != rMa.Cols()) Init(rMa.Cols()); // automatically resize. + for(size_t i=0;i<mDim;i++) + mpData[i] = rMa(i,col); // can't do this efficiently so don't really bother. + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + _ElemT + Vector<_ElemT>:: + Sum() const + { + //note the double accumulator + double sum = 0.0; + + for (size_t i = 0; i < mDim; ++i) { + sum += mpData[i]; + } + return (_ElemT)sum; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + AddColSum(const Matrix<_ElemT>& rM) + { + // note the double accumulator + double sum; + + assert(mDim == rM.Cols()); + + for (size_t i = 0; i < mDim; ++i) { + sum = 0.0; + for (size_t j = 0; j < rM.Rows(); ++j) { + sum += rM[j][i]; + } + mpData[i] += sum; + } + return *this; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + AddRowSum(const Matrix<_ElemT>& rM) + { + // note the double accumulator + double sum; + + assert(mDim == rM.Rows()); + + for (size_t i = 0; i < mDim; ++i) { + sum = 0.0; + for (size_t j = 0; j < rM.Cols(); ++j) { + sum += rM[i][j]; + } + mpData[i] += sum; + } + return *this; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + _ElemT + Vector<_ElemT>:: + LogSumExp() const + { + double sum = LOG_0; + + for (size_t i = 0; i < mDim; ++i) { + sum = LogAdd(sum, mpData[i]); + } + return sum; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + Invert() { + for (size_t i = 0; i < mDim; ++i) { + mpData[i] = static_cast<_ElemT>(1 / mpData[i]); + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + ApplyLog() { + for (size_t i = 0; i < mDim; ++i) { + mpData[i] = _LOG(mpData[i]); + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + ApplyLog(const Vector<_ElemT>& rV) { + assert(mDim==rV.Dim()); + for (size_t i = 0; i < mDim; ++i) { + mpData[i] = log(rV[i]); + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + ApplyExp() { + for (size_t i = 0; i < mDim; ++i) { + mpData[i] = _EXP(mpData[i]); + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + ApplySoftMax() { + _ElemT lse = LogSumExp(); + + for (size_t i = 0; i < mDim; ++i) { + mpData[i] = exp(mpData[i] - lse); + } + return *this; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + Add(_ElemT c) + { + for(size_t i = 0; i < mDim; i++) { + mpData[i] += c; + } + return *this; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + Subtract(_ElemT c) + { + for(size_t i = 0; i < mDim; i++) { + mpData[i] -= c; + } + return *this; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + Scale(_ElemT c) + { + for(size_t i = 0; i < mDim; i++) { + mpData[i] *= c; + } + return *this; + } + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + MultiplyElements(const Vector<_ElemT>& rV) + { + assert(mDim == rV.Dim()); + for(size_t i = 0; i < mDim; i++) { + mpData[i] *= rV[i]; + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + MultiplyElements(_ElemT alpha, const Vector<_ElemT>& rV, const Vector<_ElemT>& rR, _ElemT beta) + { + assert((mDim == rV.Dim() && mDim == rR.Dim())); + for(size_t i = 0; i < mDim; i++) { + mpData[i] = alpha * rV[i] * rR[i] + beta * mpData[i]; + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + DivideElements(const Vector<_ElemT>& rV) + { + assert(mDim == rV.Dim()); + for(size_t i = 0; i < mDim; i++) { + mpData[i] /= rV[i]; + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + + template<typename _ElemT> + Vector<_ElemT>& + Vector<_ElemT>:: + DivideElements(_ElemT alpha, const Vector<_ElemT>& rV, const Vector<_ElemT>& rR, _ElemT beta) + { + assert((mDim == rV.Dim() && mDim == rR.Dim())); + for(size_t i = 0; i < mDim; i++) { + mpData[i] = alpha * rV[i]/rR[i] + beta * mpData[i] ; + } + return *this; + } + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + void Load(std::istream& rIn, Vector<_ElemT>& rV) + { + std::streamoff pos = rIn.tellg(); + if(MatrixVectorIostreamControl::Flags(rIn, ACCUMULATE_INPUT)) { + for (size_t i = 0; i < rV.Dim(); i++) { + _ElemT tmp; + rIn >> tmp; + rV[i] += tmp; + } + } else { + for (size_t i = 0; i < rV.Dim(); i++) { + rIn >> rV[i]; + } + } + if(rIn.fail()) { + throw std::runtime_error("Failed to read vector from stream. File position is "+to_string(pos)); + } + } + + template<typename _ElemT> + std::istream & + operator >> (std::istream& rIn, Vector<_ElemT>& rV) + { + rIn >> std::ws; + if(rIn.peek() == 'v'){ // "new" format: v <dim> 1.0 0.2 4.3 ... + rIn.get(); + long long int tmp=-1; + rIn >> tmp; + if(rIn.fail() || tmp<0) { + throw std::runtime_error("Failed to read vector from stream: no size"); + } + size_t tmp2 = size_t(tmp); + assert((long long int)tmp2 == tmp); + + if(rV.Dim() != tmp2) rV.Init(tmp2); + } + Load(rIn,rV); + return rIn; + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + void Save (std::ostream& rOut, const Vector<_ElemT>& rV) + { + + for (size_t i = 0; i < rV.Dim(); i++) { + rOut << rV[i] << ' '; + } + if(rOut.fail()) { + throw std::runtime_error("Failed to write vector to stream"); + } + } + + + //**************************************************************************** + //**************************************************************************** + template<typename _ElemT> + std::ostream & + operator << (std::ostream& rOut, const Vector<_ElemT>& rV) + { + rOut << "v " << rV.Dim() << " "; + Save(rOut,rV); + return rOut; + } + + + + //**************************************************************************** + //**************************************************************************** + +#ifdef HAVE_ATLAS + template<> + float + BlasDot<>(const Vector<float>& rA, const Vector<float>& rB); + + template<> + double + BlasDot<>(const Vector<double>& rA, const Vector<double>& rB); + + template<typename _ElemT> + inline Vector<_ElemT>& + Vector<_ElemT>:: + DotMul(const Vector<_ElemT> &rV){ + assert(mDim == rV.mDim); + const _ElemT *other_data = rV.pData(); + _ElemT *my_data = mpData, *my_data_end = my_data+mDim; + for(;my_data<my_data_end;) *(my_data++) *= *(other_data++); + return *this; + } + + template<> + Vector<float>& + Vector<float>:: + BlasAxpy(const float alpha, const Vector<float>& rV); + + + template<> + Vector<double>& + Vector<double>:: + BlasAxpy(const double alpha, const Vector<double>& rV); + + + template<> + Vector<float>& + Vector<float>:: + BlasGemv(const float alpha, const Matrix<float>& rM, MatrixTrasposeType trans, const Vector<float>& rV, const float beta); + + template<> + Vector<double>& + Vector<double>:: + BlasGemv(const double alpha, const Matrix<double>& rM, MatrixTrasposeType trans, const Vector<double>& rV, const double beta); + +#else + #error Routines in this section are not implemented yet without BLAS +#endif + + + template<class _ElemT> + _ElemT + InnerProduct(const Vector<_ElemT> &v1, const Matrix<_ElemT> &M, const Vector<_ElemT> &v2){ + assert(v1.size()==M.Rows() && v2.size()==M.Cols()); + Vector<_ElemT> vtmp(M.Rows()); + vtmp.BlasGemv(1.0, M, NO_TRANS, v2, 0.0); + return BlasDot(v1, vtmp); + } + + +} // namespace TNet + + +#endif // TNet_Vector_tcc diff --git a/tnet_io/KaldiLib/clapack.cc b/tnet_io/KaldiLib/clapack.cc new file mode 100644 index 0000000..a486bef --- /dev/null +++ b/tnet_io/KaldiLib/clapack.cc @@ -0,0 +1,61 @@ + +extern "C" { + + /** + * Wrapper to GotoBLAS lapack for STK and TNet (sgetrf sgetri dgetrf dgetri) + */ + typedef float real; + typedef double doublereal; + typedef int integer; + + + /** + * The lapack interface (used in gotoblas) + */ + /* Subroutine */ int sgetrf_(integer *m, integer *n, real *a, integer *lda, + integer *ipiv, integer *info); + /* Subroutine */ int sgetri_(integer *n, real *a, integer *lda, integer *ipiv, + real *work, integer *lwork, integer *info); + /* Subroutine */ int dgetrf_(integer *m, integer *n, doublereal *a, integer * + lda, integer *ipiv, integer *info); + /* Subroutine */ int dgetri_(integer *n, doublereal *a, integer *lda, integer + *ipiv, doublereal *work, integer *lwork, integer *info); + + + + + + /** + * The clapack interface as used by ATLAS (used in STK, + */ + enum CBLAS_ORDER {CblasRowMajor=101, CblasColMajor=102 }; + + int clapack_sgetrf(const enum CBLAS_ORDER Order, const int M, const int N, + float *A, const int lda, int *ipiv) + { + return sgetrf_((int*)&M, (int*)&N, A, (int*)&lda, (int*)ipiv, 0); + } + + + int clapack_sgetri(const enum CBLAS_ORDER Order, const int N, float *A, + const int lda, const int *ipiv) + { + return sgetri_((int*)&N, A, (int*)&lda, (int*)ipiv, 0, 0, 0); + } + + + int clapack_dgetrf(const enum CBLAS_ORDER Order, const int M, const int N, + double *A, const int lda, int *ipiv) + { + return dgetrf_((int*)&M, (int*)&N, A, (int*)&lda, (int*)ipiv, 0); + } + + + int clapack_dgetri(const enum CBLAS_ORDER Order, const int N, double *A, + const int lda, const int *ipiv) + { + return dgetri_((int*)&N, A, (int*)&lda, (int*)ipiv, 0, 0, 0); + } + + +} diff --git a/tnet_io/KaldiLib/clapack.h b/tnet_io/KaldiLib/clapack.h new file mode 100644 index 0000000..0c6855d --- /dev/null +++ b/tnet_io/KaldiLib/clapack.h @@ -0,0 +1,149 @@ +/* + * Automatically Tuned Linear Algebra Software v3.8.2 + * (C) Copyright 1999 R. Clint Whaley + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions, and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the ATLAS group or the names of its contributers may + * not be used to endorse or promote products derived from this + * software without specific written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ATLAS GROUP OR ITS CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef CLAPACK_H + +#define CLAPACK_H +#include "cblas.h" + +#ifndef ATLAS_ORDER + #define ATLAS_ORDER CBLAS_ORDER +#endif +#ifndef ATLAS_UPLO + #define ATLAS_UPLO CBLAS_UPLO +#endif +#ifndef ATLAS_DIAG + #define ATLAS_DIAG CBLAS_DIAG +#endif +int clapack_sgesv(const enum CBLAS_ORDER Order, const int N, const int NRHS, + float *A, const int lda, int *ipiv, + float *B, const int ldb); +int clapack_sgetrf(const enum CBLAS_ORDER Order, const int M, const int N, + float *A, const int lda, int *ipiv); +int clapack_sgetrs + (const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE Trans, + const int N, const int NRHS, const float *A, const int lda, + const int *ipiv, float *B, const int ldb); +int clapack_sgetri(const enum CBLAS_ORDER Order, const int N, float *A, + const int lda, const int *ipiv); +int clapack_sposv(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, const int NRHS, float *A, const int lda, + float *B, const int ldb); +int clapack_spotrf(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, float *A, const int lda); +int clapack_spotrs(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const int NRHS, const float *A, const int lda, + float *B, const int ldb); +int clapack_spotri(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, float *A, const int lda); +int clapack_slauum(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, float *A, const int lda); +int clapack_strtri(const enum ATLAS_ORDER Order,const enum ATLAS_UPLO Uplo, + const enum ATLAS_DIAG Diag,const int N, float *A, const int lda); + +int clapack_dgesv(const enum CBLAS_ORDER Order, const int N, const int NRHS, + double *A, const int lda, int *ipiv, + double *B, const int ldb); +int clapack_dgetrf(const enum CBLAS_ORDER Order, const int M, const int N, + double *A, const int lda, int *ipiv); +int clapack_dgetrs + (const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE Trans, + const int N, const int NRHS, const double *A, const int lda, + const int *ipiv, double *B, const int ldb); +int clapack_dgetri(const enum CBLAS_ORDER Order, const int N, double *A, + const int lda, const int *ipiv); +int clapack_dposv(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, const int NRHS, double *A, const int lda, + double *B, const int ldb); +int clapack_dpotrf(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, double *A, const int lda); +int clapack_dpotrs(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const int NRHS, const double *A, const int lda, + double *B, const int ldb); +int clapack_dpotri(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, double *A, const int lda); +int clapack_dlauum(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, double *A, const int lda); +int clapack_dtrtri(const enum ATLAS_ORDER Order,const enum ATLAS_UPLO Uplo, + const enum ATLAS_DIAG Diag,const int N, double *A, const int lda); + +int clapack_cgesv(const enum CBLAS_ORDER Order, const int N, const int NRHS, + void *A, const int lda, int *ipiv, + void *B, const int ldb); +int clapack_cgetrf(const enum CBLAS_ORDER Order, const int M, const int N, + void *A, const int lda, int *ipiv); +int clapack_cgetrs + (const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE Trans, + const int N, const int NRHS, const void *A, const int lda, + const int *ipiv, void *B, const int ldb); +int clapack_cgetri(const enum CBLAS_ORDER Order, const int N, void *A, + const int lda, const int *ipiv); +int clapack_cposv(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, const int NRHS, void *A, const int lda, + void *B, const int ldb); +int clapack_cpotrf(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, void *A, const int lda); +int clapack_cpotrs(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const int NRHS, const void *A, const int lda, + void *B, const int ldb); +int clapack_cpotri(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, void *A, const int lda); +int clapack_clauum(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, void *A, const int lda); +int clapack_ctrtri(const enum ATLAS_ORDER Order,const enum ATLAS_UPLO Uplo, + const enum ATLAS_DIAG Diag,const int N, void *A, const int lda); + +int clapack_zgesv(const enum CBLAS_ORDER Order, const int N, const int NRHS, + void *A, const int lda, int *ipiv, + void *B, const int ldb); +int clapack_zgetrf(const enum CBLAS_ORDER Order, const int M, const int N, + void *A, const int lda, int *ipiv); +int clapack_zgetrs + (const enum CBLAS_ORDER Order, const enum CBLAS_TRANSPOSE Trans, + const int N, const int NRHS, const void *A, const int lda, + const int *ipiv, void *B, const int ldb); +int clapack_zgetri(const enum CBLAS_ORDER Order, const int N, void *A, + const int lda, const int *ipiv); +int clapack_zposv(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, const int NRHS, void *A, const int lda, + void *B, const int ldb); +int clapack_zpotrf(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, void *A, const int lda); +int clapack_zpotrs(const enum CBLAS_ORDER Order, const enum CBLAS_UPLO Uplo, + const int N, const int NRHS, const void *A, const int lda, + void *B, const int ldb); +int clapack_zpotri(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, void *A, const int lda); +int clapack_zlauum(const enum ATLAS_ORDER Order, const enum ATLAS_UPLO Uplo, + const int N, void *A, const int lda); +int clapack_ztrtri(const enum ATLAS_ORDER Order,const enum ATLAS_UPLO Uplo, + const enum ATLAS_DIAG Diag,const int N, void *A, const int lda); + +#endif diff --git a/tnet_io/Makefile b/tnet_io/Makefile new file mode 100644 index 0000000..9389c8c --- /dev/null +++ b/tnet_io/Makefile @@ -0,0 +1,13 @@ +.PHONY: tnet +all: tnet $(OBJ_DIR)/test +$(OBJ_DIR)/test: $(OBJ_DIR)/cwrapper.o $(OBJ_DIR)/test.o $(OBJ_DIR)/libKaldiLib.a + g++ -o $@ $^ -Wl,-rpath=../../build/objs/ -L../../build/objs/ -lnerv -Wl,-rpath=../../build/luajit-2.0/lib -L../../build/luajit-2.0/lib -lluajit-5.1 +$(OBJ_DIR)/cwrapper.o: cwrapper.cpp + g++ -o $@ -c $< -DHAVE_ATLAS -I KaldiLib/ -g +$(OBJ_DIR)/%.o: %.c + g++ -o $@ -c $< -g +clean: + -rm $(OBJ_DIR)/*.o + $(MAKE) -C KaldiLib/ clean +tnet: + $(MAKE) -C KaldiLib/ diff --git a/tnet_io/cwrapper.cpp b/tnet_io/cwrapper.cpp new file mode 100644 index 0000000..f992731 --- /dev/null +++ b/tnet_io/cwrapper.cpp @@ -0,0 +1,136 @@ +#include "KaldiLib/Features.h" +#include "KaldiLib/Labels.h" +#include "KaldiLib/Common.h" +#include "KaldiLib/UserInterface.h" +#include <string> +#define SNAME "TNET" + +extern "C" { +#include "cwrapper.h" +#include "string.h" + + extern Matrix *nerv_matrix_host_float_new_(long nrow, long ncol); + + struct TNetFeatureRepo { + TNet::FeatureRepository feature_repo; + TNet::UserInterface ui; + bool swap_features; + int target_kind; + int deriv_order; + int* p_deriv_win_lenghts; + int start_frm_ext; + int end_frm_ext; + char* cmn_path; + char* cmn_file; + const char* cmn_mask; + char* cvn_path; + char* cvn_file; + const char* cvn_mask; + const char* cvg_file; + TNet::Matrix<float> feats_host; /* KaldiLib implementation */ + }; + + TNetFeatureRepo *tnet_feature_repo_new(const char *p_script, const char *config, int context) { + TNetFeatureRepo *repo = new TNetFeatureRepo(); + repo->ui.ReadConfig(config); + fprintf(stderr, "read config\n"); + repo->swap_features = !repo->ui.GetBool(SNAME":NATURALREADORDER", TNet::IsBigEndian()); + /* load defaults */ + repo->target_kind = repo->ui.GetFeatureParams(&repo->deriv_order, + &repo->p_deriv_win_lenghts, + &repo->start_frm_ext, &repo->end_frm_ext, + &repo->cmn_path, &repo->cmn_file, &repo->cmn_mask, + &repo->cvn_path, &repo->cvn_file, &repo->cvn_mask, + &repo->cvg_file, SNAME":", 0); + repo->start_frm_ext = repo->end_frm_ext = context; + repo->feature_repo.Init(repo->swap_features, + repo->start_frm_ext, repo->end_frm_ext, repo->target_kind, + repo->deriv_order, repo->p_deriv_win_lenghts, + repo->cmn_path, repo->cmn_mask, + repo->cvn_path, repo->cvn_mask, repo->cvg_file); + repo->feature_repo.AddFileList(p_script); + repo->feature_repo.Rewind(); + return repo; + } + + Matrix *tnet_feature_repo_read_utterance(TNetFeatureRepo *repo) { + Matrix *mat; /* nerv implementation */ + repo->feature_repo.ReadFullMatrix(repo->feats_host); + std::string utter_str = repo->feature_repo.Current().Logical(); + repo->feats_host.CheckData(utter_str); + int n = repo->feats_host.Rows(); + int m = repo->feats_host.Cols(); + mat = nerv_matrix_host_float_new_(n, m); + size_t stride = mat->stride; + fprintf(stderr, "[tnet] feature: %s %d %d\n", utter_str.c_str(), n, m); + for (int i = 0; i < n; i++) + { + float *row = repo->feats_host.pRowData(i); + float *nerv_row = (float *)((char *)mat->data.f + i * stride); + /* use memmove to copy the row, since KaldiLib uses compact storage */ + memmove(nerv_row, row, sizeof(float) * m); + } + return mat; + } + + void tnet_feature_repo_next(TNetFeatureRepo *repo) { + repo->feature_repo.MoveNext(); + } + + size_t tnet_feature_repo_current_samplerate(TNetFeatureRepo *repo) { + return repo->feature_repo.CurrentHeader().mSamplePeriod; + } + + const char *tnet_feature_repo_current_tag(TNetFeatureRepo *repo) { + return repo->feature_repo.Current().Logical().c_str(); + } + + void tnet_feature_repo_destroy(TNetFeatureRepo *repo) { + if (repo->cmn_mask) + free(repo->cmn_path); + if (repo->cvn_mask) + free(repo->cvn_path); + free(repo->p_deriv_win_lenghts); + free(repo); + } + + struct TNetLabelRepo { + TNet::LabelRepository label_repo; + }; + + TNetLabelRepo *tnet_label_repo_new(const char *mlf, const char *fmt, + const char *fmt_arg, const char *dir, + const char *ext) { + TNetLabelRepo *repo = new TNetLabelRepo(); + repo->label_repo.InitExt(mlf, fmt, fmt_arg, dir, ext); + /* repo->label_repo.Init(mlf, fmt_arg, dir, ext); */ + return repo; + } + + Matrix *tnet_label_repo_read_utterance(TNetLabelRepo *repo, + size_t frames, + size_t sample_rate, + const char *tag) { + std::vector<TNet::Matrix<float> > labs_hosts; /* KaldiLib implementation */ + Matrix *mat; + repo->label_repo.GenDesiredMatrixExt(labs_hosts, frames, + sample_rate, tag); + int n = labs_hosts[0].Rows(); + int m = labs_hosts[0].Cols(); + mat = nerv_matrix_host_float_new_(n, m); + size_t stride = mat->stride; + fprintf(stderr, "[tnet] label: %s %d %d\n", tag, n, m); + for (int i = 0; i < n; i++) + { + float *row = labs_hosts[0].pRowData(i); + float *nerv_row = (float *)((char *)mat->data.f + i * stride); + /* use memmove to copy the row, since KaldiLib uses compact storage */ + memmove(nerv_row, row, sizeof(float) * m); + } + return mat; + } + + void tnet_label_repo_destroy(TNetLabelRepo *repo) { + free(repo); + } +} diff --git a/tnet_io/cwrapper.h b/tnet_io/cwrapper.h new file mode 100644 index 0000000..7a9449a --- /dev/null +++ b/tnet_io/cwrapper.h @@ -0,0 +1,33 @@ +#ifndef NERV_TNET_IO_CWRAPPER +#define NERV_TNET_IO_CWRAPPER +#include "../../matrix/generic/matrix.h" +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct TNetFeatureRepo TNetFeatureRepo; + + TNetFeatureRepo *tnet_feature_repo_new(const char *scp, + const char *config, int context); + Matrix *tnet_feature_repo_read_utterance(TNetFeatureRepo *repo); + size_t tnet_feature_repo_current_samplerate(TNetFeatureRepo *repo); + const char *tnet_feature_repo_current_tag(TNetFeatureRepo *repo); + void tnet_feature_repo_next(TNetFeatureRepo *repo); + void tnet_feature_repo_destroy(TNetFeatureRepo *repo); + + typedef struct TNetLabelRepo TNetLabelRepo; + + TNetLabelRepo *tnet_label_repo_new(const char *mlf, const char *fmt, + const char *fmt_arg, const char *dir, + const char *ext); + + Matrix *tnet_label_repo_read_utterance(TNetLabelRepo *repo, + size_t frames, + size_t sample_rate, + const char *tag); + + void tnet_label_repo_destroy(TNetLabelRepo *repo); +#ifdef __cplusplus +} +#endif +#endif diff --git a/tnet_io/test.c b/tnet_io/test.c new file mode 100644 index 0000000..397b249 --- /dev/null +++ b/tnet_io/test.c @@ -0,0 +1,41 @@ +#include "cwrapper.h" +#include <stdio.h> + +extern "C" { + void print_nerv_matrix(Matrix *mat) { + int n = mat->nrow; + int m = mat->ncol; + int i, j; + size_t stride = mat->stride; + for (i = 0; i < n; i++) + { + + float *nerv_row = (float *)((char *)mat->data.f + i * stride); + for (j = 0; j < m; j++) + fprintf(stderr, "%.8f ", nerv_row[j]); + fprintf(stderr, "\n"); + } + } + + int main() { + fprintf(stderr, "init repo\n"); + TNetFeatureRepo *feat_repo = tnet_feature_repo_new( + "/slfs1/users/mfy43/swb_ivec/train_bp.scp", + "/slfs1/users/mfy43/swb_ivec/plp_0_d_a.conf", 5); + Matrix *feat_utter; + feat_utter = tnet_feature_repo_read_utterance(feat_repo); + + TNetLabelRepo *lab_repo = tnet_label_repo_new( + "/slfs1/users/mfy43/swb_ivec/ref.mlf", + "map", + "/slfs1/users/mfy43/swb_ivec/dict", + "*/", + "lab"); + Matrix *lab_utter = tnet_label_repo_read_utterance(lab_repo, + feat_utter->nrow - 5 * 2, + tnet_feature_repo_current_samplerate(feat_repo), + tnet_feature_repo_current_tag(feat_repo)); + print_nerv_matrix(lab_utter); + return 0; + } +} diff --git a/tnet_io/tnet.mk b/tnet_io/tnet.mk new file mode 100644 index 0000000..775fe85 --- /dev/null +++ b/tnet_io/tnet.mk @@ -0,0 +1,83 @@ +# +# This makefile contains some global definitions, +# that are used during the build process. +# It is included by all the subridrectory libraries. +# + + +############################################################## +##### 64-BIT CROSS-COMPILATION ##### +CXXFLAGS= +FWDPARAM= +BITS64=true +ifeq ($(BITS64), true) + ##### CHANGE WHEN DIFFERENT 64BIT g++ PREFIX ##### + CROSS_COMPILE = x86_64-linux- + ##### CHANGE WHEN DIFFERENT 64BIT g++ PREFIX ##### + CXXFLAGS += -m64 + FWDPARAM += BITS64=true +else + CXXFLAGS += -m32 +endif + +# disable cross-compile prefix if CXX not exists +CXX=$(CROSS_COMPILE)g++ +CXX2=$(notdir $(shell which $(CXX) 2>/dev/null)) +ifneq ("$(CXX)", "$(CXX2)") + CROSS_COMPILE= +endif + +# compilation tools +CC = $(CROSS_COMPILE)g++ +CXX = $(CROSS_COMPILE)g++ +AR = $(CROSS_COMPILE)ar +RANLIB = $(CROSS_COMPILE)ranlib +AS = $(CROSS_COMPILE)as + + + + +############################################################## +##### PATH TO CUDA TOOLKIT ##### +#CUDA_TK_BASE=/usr/local/share/cuda-3.2.12 +CUDA_TK_BASE=/usr/local/cuda +##### PATH TO CUDA TOOLKIT ##### + + + + +# compilation args +CXXFLAGS += -g -Wall -O2 -DHAVE_ATLAS -rdynamic +CXXFLAGS += -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings -Wconversion + +# enable double-precision +ifeq ($(DOUBLEPRECISION), true) + CXXFLAGS += -DDOUBLEPRECISION + FWDPARAM += DOUBLEPRECISION=true +endif + + +# compile all the source .cc files +SRC=$(wildcard *.cc) +OBJ=$(addprefix $(OBJ_DIR)/,$(patsubst %.cc, %.o, $(SRC))) + + + + +######################################################### +# CONFIGURATION CHECKS +# + +#check that CUDA_TK_BASE is set correctly +ifeq ("$(wildcard $(CUDA_TK_BASE)/bin/nvcc)", "$(CUDA_TK_BASE)/bin/nvcc") + HAVE_CUDA=true +else + ifeq ($(CUDA), true) + $(error %%% CUDA not found! Incorrect path in CUDA_TK_BASE: $(CUDA_TK_BASE) in 'trunk/src/tnet.mk') + endif +endif + +# +######################################################### + + |