-#include <string>

-#include <stdexcept>

-#include <cmath>

-#include <cfloat>

-#include <cstdio>

-

-#include "Common.h"

-#include "MathAux.h"

-

-

-/// Defines the white chars for string trimming

-#if !defined(WHITE_CHARS)

-# define WHITE_CHARS " \t"

-#endif

-

-namespace TNet {

-

-#include <ios>

-

- // Allocating stream variable used by stream modifier MatrixVectorIostreamControl

- const int MATRIX_IOS_FORMAT_IWORD = std::ios_base::xalloc();

-

- //***************************************************************************

- //***************************************************************************

- int getHTKstr(char *str)

- {

- char termChar = '\0';

- char *chrptr = str;

-

- while (std::isspace(*chrptr)) ++chrptr;

-

- if (*chrptr == '\'' || *chrptr == '"') {

- termChar = *chrptr;

- chrptr++;

- }

-

- for (; *chrptr; chrptr++) {

- if (*chrptr == '\'' || *chrptr == '"') {

- if (termChar == *chrptr) {

- termChar = '\0';

- chrptr++;

- break;

- }

- }

-

- if (std::isspace(*chrptr) && !termChar) {

- break;

- }

-

- if (*chrptr == '\\') {

- ++chrptr;

- if (*chrptr == '\0' || (*chrptr >= '0' && *chrptr <= '7' &&

- (*++chrptr < '0' || *chrptr > '7' ||

- *++chrptr < '0' || *chrptr > '7'))) {

- return -1;

- }

-

- if (*chrptr >= '0' && *chrptr <= '7') {

- *chrptr = (char)((*chrptr - '0') + (chrptr[-1] - '0') * 8 + (chrptr[-2] - '0') * 64);

- }

- }

- *str++ = *chrptr;

- }

-

- if (termChar) {

- return -2;

- }

-

- *str = '\0';

-

- return 0;

- }

-

-

- //*****************************************************************************

- //*****************************************************************************

- void

- ParseHTKString(const std::string & rIn, std::string & rOut)

- {

- int ret_val;

-

- // the new string will be at most as long as the original, so we allocate

- // space

- char* new_str = new char[rIn.size() + 1];

-

- char* p_htk_str = new_str;

-

- strcpy(p_htk_str, rIn.c_str());

- ret_val = getHTKstr(p_htk_str);

-

- // call the function

- if (!ret_val) {

- rOut = p_htk_str;

- }

-

- delete [] new_str;

-

- if (ret_val) {

- throw std::runtime_error("Error parsing HTK string");

- }

- }

-

-

-

- //***************************************************************************

- //***************************************************************************

- bool

- IsBigEndian()

- {

- int a = 1;

- return (bool) ((char *) &a)[0] != 1;

- }

-

-

- //***************************************************************************

- //***************************************************************************

- void

- MakeHtkFileName(char* pOutFileName, const char* inFileName,

- const char* out_dir, const char* out_ext)

- {

- const char* base_name;

- const char* bname_end = NULL;

- const char* chrptr;

-

- // if (*inFileName == '*' && *++inFileName == '/') ++inFileName;

-

- // we don't do anything if file is stdin/out

- if (!strcmp(inFileName, "-"))

- {

- pOutFileName[0] = '-';

- pOutFileName[1] = '\0';

- return;

- }

-

- base_name = strrchr(inFileName, '/');

- base_name = base_name != NULL ? base_name + 1 : inFileName;

-

- if (out_ext) bname_end = strrchr(base_name, '.');

- if (!bname_end) bname_end = base_name + strlen(base_name);

-

-

- if ((chrptr = strstr(inFileName, "/./")) != NULL)

- {

- // what is in path after /./ serve as base name

- base_name = chrptr + 3;

- }

- /* else if (*inFileName != '/')

- {

- // if inFileName isn't absolut path, don't forget directory structure

- base_name = inFileName;

- }*/

-

- *pOutFileName = '\0';

- if (out_dir)

- {

- if (*out_dir)

- {

- strcat(pOutFileName, out_dir);

- strcat(pOutFileName, "/");

- }

- strncat(pOutFileName, base_name, bname_end-base_name);

- }

- else

- {

- strncat(pOutFileName, inFileName, bname_end-inFileName);

- }

-

- if (out_ext && *out_ext)

- {

- strcat(pOutFileName, ".");

- strcat(pOutFileName, out_ext);

- }

- }

-

-

- //****************************************************************************

- //****************************************************************************

- bool

- CloseEnough(const float f1, const float f2, const float nRounds)

- {

- bool ret_val = (_ABS((f1 - f2) / (f2 == 0.0f ? 1.0f : f2))

- < (nRounds * FLT_EPSILON));

-

- return ret_val;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- bool

- CloseEnough(const double f1, const double f2, const double nRounds)

- {

- bool ret_val = (_ABS((f1 - f2) / (f2 == 0.0 ? 1.0 : f2))

- < (nRounds * DBL_EPSILON));

-

- return ret_val;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- char*

- ExpandHtkFilterCmd(const char *command, const char *filename, const char* pFilter)

- {

-

- char *out, *outend;

- const char *chrptr = command;

- int ndollars = 0;

- int fnlen = strlen(filename);

-

- while (*chrptr++) ndollars += (*chrptr == *pFilter);

-

- out = (char*) malloc(strlen(command) - ndollars + ndollars * fnlen + 1);

-

- outend = out;

-

- for (chrptr = command; *chrptr; chrptr++) {

- if (*chrptr == *pFilter) {

- strcpy(outend, filename);

- outend += fnlen;

- } else {

- *outend++ = *chrptr;

- }

- }

- *outend = '\0';

- return out;

- }

-

- //***************************************************************************

- //***************************************************************************

- char *

- StrToUpper(char *str)

- {

- char *chptr;

- for (chptr = str; *chptr; chptr++) {

- *chptr = (char)toupper(*chptr);

- }

- return str;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- std::string&

- Trim(std::string& rStr)

- {

- // WHITE_CHARS is defined in common.h

- std::string::size_type pos = rStr.find_last_not_of(WHITE_CHARS);

- if(pos != std::string::npos)

- {

- rStr.erase(pos + 1);

- pos = rStr.find_first_not_of(WHITE_CHARS);

- if(pos != std::string::npos) rStr.erase(0, pos);

- }

- else

- rStr.erase(rStr.begin(), rStr.end());

-

- return rStr;

- }

-

-

-} // namespace TNet

-

-//#ifdef CYGWIN

-

-void assertf(const char *c, int i, const char *msg){

- printf("Assertion \"%s\" failed: file \"%s\", line %d\n", msg?msg:"(null)", c?c:"(null)", i);

- abort();

-}

-

-

-void assertf_throw(const char *c, int i, const char *msg){

- char buf[2000];

- snprintf(buf, 1999, "Assertion \"%s\" failed, throwing exception: file \"%s\", line %d\n", msg?msg:"(null)", c?c:"(null)", i);

- throw std::runtime_error((std::string)buf);

-}

-//#endif

-#ifndef TNet_Common_h

-#define TNet_Common_h

-

-#include <cstdlib>

-#include <string.h> // C string stuff like strcpy

-#include <string>

-#include <sstream>

-#include <stdexcept>

-

-/* Alignment of critical dynamic data structure

- *

- * Not all platforms support memalign so we provide a stk_memalign wrapper

- * void *stk_memalign( size_t align, size_t size, void **pp_orig )

- * *pp_orig is the pointer that has to be freed afterwards.

- */

-#ifdef HAVE_POSIX_MEMALIGN

-# define stk_memalign(align,size,pp_orig) \

- ( !posix_memalign( pp_orig, align, size ) ? *(pp_orig) : NULL )

-# ifdef STK_MEMALIGN_MANUAL

-# undef STK_MEMALIGN_MANUAL

-# endif

-#elif defined(HAVE_MEMALIGN)

- /* Some systems have memalign() but no declaration for it */

- //void * memalign( size_t align, size_t size );

-# define stk_memalign(align,size,pp_orig) \

- ( *(pp_orig) = memalign( align, size ) )

-# ifdef STK_MEMALIGN_MANUAL

-# undef STK_MEMALIGN_MANUAL

-# endif

-#else /* We don't have any choice but to align manually */

-# define stk_memalign(align,size,pp_orig) \

- (( *(pp_orig) = malloc( size + align - 1 )) ? \

- (void *)( (((unsigned long)*(pp_orig)) + 15) & ~0xFUL ) : NULL )

-# define STK_MEMALIGN_MANUAL

-#endif

-

-

-#define swap8(a) { \

- char t=((char*)&a)[0]; ((char*)&a)[0]=((char*)&a)[7]; ((char*)&a)[7]=t;\

- t=((char*)&a)[1]; ((char*)&a)[1]=((char*)&a)[6]; ((char*)&a)[6]=t;\

- t=((char*)&a)[2]; ((char*)&a)[2]=((char*)&a)[5]; ((char*)&a)[5]=t;\

- t=((char*)&a)[3]; ((char*)&a)[3]=((char*)&a)[4]; ((char*)&a)[4]=t;}

-#define swap4(a) { \

- char t=((char*)&a)[0]; ((char*)&a)[0]=((char*)&a)[3]; ((char*)&a)[3]=t;\

- t=((char*)&a)[1]; ((char*)&a)[1]=((char*)&a)[2]; ((char*)&a)[2]=t;}

-#define swap2(a) { \

- char t=((char*)&a)[0]; ((char*)&a)[0]=((char*)&a)[1]; ((char*)&a)[1]=t;}

-

-

-namespace TNet

-{

- /** **************************************************************************

- ** **************************************************************************

- * @brief Aligns a number to a specified base

- * @param n Number of type @c _T to align

- * @return Aligned value of type @c _T

- */

- template<size_t _align, typename _T>

- inline _T

- align(const _T n)

- {

- const _T x(_align - 1);

- return (n + x) & ~(x);

- }

-

-

- /**

- * @brief Returns true if architecture is big endian

- */

- bool

- IsBigEndian();

-

-

- /**

- * @brief Returns true if two numbers are close enough to each other

- *

- * @param f1 First operand

- * @param f2 Second operand

- * @param nRounds Expected number of operations prior to this comparison

- */

- bool

- CloseEnough(const float f1, const float f2, const float nRounds);

-

-

- /**

- * @brief Returns true if two numbers are close enough to each other

- *

- * @param f1 First operand

- * @param f2 Second operand

- * @param nRounds Expected number of operations prior to this comparison

- */

- bool

- CloseEnough(const double f1, const double f2, const double nRounds);

-

-

- /**

- * @brief Parses a HTK-style string into a C++ std::string readable

- *

- * @param rIn HTK input string

- * @param rOut output parsed string

- */

- void

- ParseHTKString(const std::string & rIn, std::string & rOut);

-

-

- /**

- * @brief Synthesize new file name based on name, path, and extension

- *

- * @param pOutFileName full ouptut file name

- * @param pInFileName file name

- * @param pOutDir directory

- * @param pOutExt extension

- */

- void

- MakeHtkFileName(char *pOutFileName, const char* pInFileName, const char *pOutDir,

- const char *pOutExt);

-

-

- /**

- * @brief Removes the leading and trailing white chars

- *

- * @param rStr Refference to the string to be processed

- * @return Refference to the original string

- *

- * The white characters are determined by the @c WHITE_CHARS macro defined

- * above.

- */

- std::string&

- Trim(std::string& rStr);

-

-

- char*

- StrToUpper(char* pStr);

-

- char*

- ExpandHtkFilterCmd(const char *command, const char *filename, const char* pFilter);

-

-

- template <class T>

- std::string to_string(const T& val)

- {

- std::stringstream ss;

- ss << val;

- return ss.str();

- }

-

- inline void

- ExpectKeyword(std::istream &i_stream, const char *kwd)

- {

- std::string token;

- i_stream >> token;

- if (token != kwd) {

- throw std::runtime_error(std::string(kwd) + " expected");

- }

- }

-

- extern const int MATRIX_IOS_FORMAT_IWORD;

-

- enum MatrixVectorIostreamControlBits {

- ACCUMULATE_INPUT = 1,

-// BINARY_OUTPUT = 2

- };

-

- class MatrixVectorIostreamControl

- {

- public:

- MatrixVectorIostreamControl(enum MatrixVectorIostreamControlBits bitsToBeSet, bool valueToBeSet)

- : mBitsToBeSet(bitsToBeSet), mValueToBeSet(valueToBeSet) {}

-

- static long Flags(std::ios_base &rIos, enum MatrixVectorIostreamControlBits bits)

- { return rIos.iword(MATRIX_IOS_FORMAT_IWORD); }

-

- long mBitsToBeSet;

- bool mValueToBeSet;

-

- friend std::ostream & operator <<(std::ostream &rOs, const MatrixVectorIostreamControl modifier)

- {

- if(modifier.mValueToBeSet) {

- rOs.iword(MATRIX_IOS_FORMAT_IWORD) |= modifier.mBitsToBeSet;

- } else {

- rOs.iword(MATRIX_IOS_FORMAT_IWORD) &= ~modifier.mBitsToBeSet;

- }

- return rOs;

- }

-

- friend std::istream & operator >>(std::istream &rIs, const MatrixVectorIostreamControl modifier)

- {

- if(modifier.mValueToBeSet) {

- rIs.iword(MATRIX_IOS_FORMAT_IWORD) |= modifier.mBitsToBeSet;

- } else {

- rIs.iword(MATRIX_IOS_FORMAT_IWORD) &= ~modifier.mBitsToBeSet;

- }

- return rIs;

- }

- };

-

-

-

-

-} // namespace TNet

-

-#ifdef __ICC

-#pragma warning (disable: 383) // ICPC remark we don't want.

-#pragma warning (disable: 810) // ICPC remark we don't want.

-#pragma warning (disable: 981) // ICPC remark we don't want.

-#pragma warning (disable: 1418) // ICPC remark we don't want.

-#pragma warning (disable: 444) // ICPC remark we don't want.

-#pragma warning (disable: 869) // ICPC remark we don't want.

-#pragma warning (disable: 1287) // ICPC remark we don't want.

-#pragma warning (disable: 279) // ICPC remark we don't want.

-#pragma warning (disable: 981) // ICPC remark we don't want.

-#endif

-

-//#ifdef CYGWIN

-#if 1

-#undef assert

-#ifndef NDEBUG

-#define assert(e) ((e) ? (void)0 : assertf(__FILE__, __LINE__, #e))

-#else

-#define assert(e) ((void)0)

-#endif

-void assertf(const char *c, int i, const char *msg); // Just make it possible to break into assert on gdb-- has some kind of bug on cygwin.

-#else

-#include <cassert>

-#endif

-

-#define assert_throw(e) ((e) ? (void)0 : assertf_throw(__FILE__, __LINE__, #e))

-void assertf_throw(const char *c, int i, const char *msg);

-

-#define DAN_STYLE_IO

-

-#endif // ifndef TNet_Common_h

-

-//

-// C++ Interface: %{MODULE}

-//

-// Description:

-//

-//

-// Author: %{AUTHOR} <%{EMAIL}>, (C) %{YEAR}

-//

-// Copyright: See COPYING file that comes with this distribution

-//

-//

-

-/** @file Error.h

- * This header defines several types and functions relating to the

- * handling of exceptions in STK.

- */

-

-#ifndef TNET_Error_h

-#define TNET_Error_h

-

-#include <iostream>

-#include <stdexcept>

-#include <string>

-#include <sstream>

-

-#include <cstdlib>

-#include <execinfo.h>

-#include <cstdarg>

-#include <cstdio>

-

-// THESE MACROS TERRIBLY CLASH WITH STK!!!!

-// WE MUST USE SAME MACROS!

-//

-//#define Error(msg) _Error_(__func__, __FILE__, __LINE__, msg)

-//#define Warning(msg) _Warning_(__func__, __FILE__, __LINE__, msg)

-//#define TraceLog(msg) _TraceLog_(__func__, __FILE__, __LINE__, msg)

-//

-

-#ifndef Error

- #define Error(...) _Error_(__func__, __FILE__, __LINE__, __VA_ARGS__)

-#endif

-#ifndef PError

- #define PError(...) _PError_(__func__, __FILE__, __LINE__, __VA_ARGS__)

-#endif

-#ifndef Warning

- #define Warning(...) _Warning_(__func__, __FILE__, __LINE__, __VA_ARGS__)

-#endif

-#ifndef TraceLog

- #define TraceLog(...) _TraceLog_(__func__, __FILE__, __LINE__, __VA_ARGS__)

-#endif

-

-namespace TNet {

-

-

-

- /** MyException

- * Custom exception class, gets the stacktrace

- */

- class MyException

- : public std::runtime_error

- {

- public:

- explicit MyException(const std::string& what_arg) throw();

- virtual ~MyException() throw();

-

- const char* what() const throw()

- { return mWhat.c_str(); }

-

- private:

- std::string mWhat;

- };

-

- /**

- * MyException:: implemenatation

- */

- inline

- MyException::

- MyException(const std::string& what_arg) throw()

- : std::runtime_error(what_arg)

- {

- mWhat = what_arg;

- mWhat += "\nTHE STACKTRACE INSIDE MyException OBJECT IS:\n";

-

- void *array[10];

- size_t size;

- char **strings;

- size_t i;

-

- size = backtrace (array, 10);

- strings = backtrace_symbols (array, size);

-

- //<< 0th string is the MyException ctor, so ignore and start by 1

- for (i = 1; i < size; i++) {

- mWhat += strings[i];

- mWhat += "\n";

- }

-

- free (strings);

- }

-

-

- inline

- MyException::

- ~MyException() throw()

- { }

-

-

- /**

- * @brief Error throwing function (with backtrace)

- */

- inline void

- _Error_(const char *func, const char *file, int line, const std::string &msg)

- {

- std::stringstream ss;

- ss << "ERROR (" << func << ':' << file << ':' << line << ") " << msg;

- throw MyException(ss.str());

- }

-

- /**

- * @brief Throw a formatted error

- */

- inline void _PError_(const char *func, const char *file, int line, const char *fmt, ...) {

- va_list ap;

- char msg[256];

- va_start(ap, fmt);

- vsnprintf(msg, sizeof msg, fmt, ap);

- va_end(ap);

- _Error_(func, file, line, msg);

- }

-

- /**

- * @brief Warning handling function

- */

- inline void

- _Warning_(const char *func, const char *file, int line, const std::string &msg)

- {

- std::cout << "WARNING (" << func << ':' << file << ':' << line << ") " << msg << std::endl;

- }

-

- inline void

- _TraceLog_(const char *func, const char *file, int line, const std::string &msg)

- {

- std::cout << "INFO (" << func << ':' << file << ':' << line << ") " << msg << std::endl;

- std::cout.flush();

- }

-

- /**

- * New kaldi error handling:

- *

- * class KaldiErrorMessage is invoked from the KALDI_ERROR macro.

- * The destructor throws an exception.

- */

- class KaldiErrorMessage {

- public:

- KaldiErrorMessage(const char *func, const char *file, int line) {

- this->stream() << "ERROR ("

- << func << "():"

- << file << ':' << line << ") ";

- }

- inline std::ostream &stream() { return ss; }

- ~KaldiErrorMessage() { throw MyException(ss.str()); }

- private:

- std::ostringstream ss;

- };

- #define KALDI_ERR TNet::KaldiErrorMessage(__func__, __FILE__, __LINE__).stream()

-

-

-

-} // namespace TNet

-

-//#define TNET_Error_h

-#endif

-

-//enable feature repository profiling

-#define PROFILING 1

-

-#include <sstream>

-#include <map>

-#include <list>

-#include <cstdio>

-

-#include "Features.h"

-#include "Tokenizer.h"

-#include "StkMatch.h"

-#include "Types.h"

-

-

-

-namespace TNet

-{

- const char

- FeatureRepository::

- mpParmKindNames[13][16] =

- {

- {"WAVEFORM"},

- {"LPC"},

- {"LPREFC"},

- {"LPCEPSTRA"},

- {"LPDELCEP"},

- {"IREFC"},

- {"MFCC"},

- {"FBANK"},

- {"MELSPEC"},

- {"USER"},

- {"DISCRETE"},

- {"PLP"},

- {"ANON"}

- };

-

- //***************************************************************************

- //***************************************************************************

-

- FileListElem::

- FileListElem(const std::string & rFileName)

- {

- std::string::size_type pos;

-

- mLogical = rFileName;

- mWeight = 1.0;

-

- // some slash-backslash replacement hack

- for (size_t i = 0; i < mLogical.size(); i++) {

- if (mLogical[i] == '\\') {

- mLogical[i] = '/';

- }

- }

-

- // read sentence weight definition if any ( physical_file.fea[s,e]{weight} )

- if ((pos = mLogical.find('{')) != std::string::npos)

- {

- std::string tmp_weight(mLogical.begin() + pos + 1, mLogical.end());

- std::stringstream tmp_ss(tmp_weight);

-

- tmp_ss >> mWeight;

- mLogical.erase(pos);

- }

-

- // look for "=" symbol and if found, split it

- if ((pos = mLogical.find('=')) != std::string::npos)

- {

- // copy all from mLogical[pos+1] till the end to mPhysical

- mPhysical.assign(mLogical.begin() + pos + 1, mLogical.end());

- // erase all from pos + 1 till the end from mLogical

- mLogical.erase(pos);

- // trim the leading and trailing spaces

- Trim(mPhysical);

- Trim(mLogical);

- }

- else

- {

- // trim the leading and trailing spaces

- Trim(mLogical);

-

- mPhysical = mLogical;

- }

- }

-

-

- //###########################################################################

- //###########################################################################

- // FeatureRepository section

- //###########################################################################

- //###########################################################################

-

- //***************************************************************************

- //***************************************************************************

- void

- FeatureRepository::

- ReadCepsNormFile(

- const char * pFileName,

- char ** pLastFileName,

- BaseFloat ** vec_buff,

- int sampleKind,

- CNFileType type,

- int coefs)

- {

- FILE* fp;

- int i;

- char s1[64];

- char s2[64];

- const char* typeStr = (type == CNF_Mean ? "MEAN" :

- type == CNF_Variance ? "VARIANCE" : "VARSCALE");

-

- const char* typeStr2 = (type == CNF_Mean ? "CMN" :

- type == CNF_Variance ? "CVN" : "VarScale");

-

- if (*pLastFileName != NULL && !strcmp(*pLastFileName, pFileName)) {

- return;

- }

- free(*pLastFileName);

- *pLastFileName=strdup(pFileName);

- *vec_buff = (BaseFloat*) realloc(*vec_buff, coefs * sizeof(BaseFloat));

-

- if (*pLastFileName == NULL || *vec_buff== NULL)

- throw std::runtime_error("Insufficient memory");

-

- if ((fp = fopen(pFileName, "r")) == NULL) {

- throw std::runtime_error(std::string("Cannot open ") + typeStr2

- + " pFileName: '" + pFileName + "'");

- }

-

- if ((type != CNF_VarScale

- && (fscanf(fp, " <%64[^>]> <%64[^>]>", s1, s2) != 2

- || strcmp(StrToUpper(s1), "CEPSNORM")

- || ReadParmKind(s2, false) != sampleKind))

- || fscanf(fp, " <%64[^>]> %d", s1, &i) != 2

- || strcmp(StrToUpper(s1), typeStr)

- || i != coefs)

- {

- ParmKind2Str(sampleKind, s2);

-

- //std::cout << "[[[TADY!!!!]]]" << pFileName << "\n" << std::flush;

-

- throw std::runtime_error(std::string("")

- + (type == CNF_VarScale ? "" : "<CEPSNORM> <")

- + (type == CNF_VarScale ? "" : s2)

- + (type == CNF_VarScale ? "" : ">")

- + " <" + typeStr + " ... expected in " + typeStr2

- + " file " + pFileName);

- }

-

- for (i = 0; i < coefs; i++) {

- if (fscanf(fp, " "FLOAT_FMT, *vec_buff+i) != 1) {

- if (fscanf(fp, "%64s", s2) == 1) {

- throw std::runtime_error(std::string("Decimal number expected but '")

- + s2 + "' found in " + typeStr2 + " file " + pFileName);

- }

- else if (feof(fp)) {

- throw std::runtime_error(std::string("Unexpected end of ")

- + typeStr2 + " file "+ pFileName);

- }

- else {

- throw std::runtime_error(std::string("Cannot read ") + typeStr2

- + " file " + pFileName);

- }

- }

-

- if (type == CNF_Variance)

- (*vec_buff)[i] = BaseFloat(1 / sqrt((*vec_buff)[i]));

- else if (type == CNF_VarScale)

- (*vec_buff)[i] = BaseFloat(sqrt((*vec_buff)[i]));

- }

-

- if (fscanf(fp, "%64s", s2) == 1)

- {

- throw std::runtime_error(std::string("End of file expected but '")

- + s2 + "' found in " + typeStr2 + " file " + pFileName);

- }

-

- fclose(fp);

- } // ReadCepsNormFile(...)

-

-

- //***************************************************************************

- //***************************************************************************

- void

- FeatureRepository::

- HtkFilter(const char* pFilter, const char* pValue, FeatureRepository& rOut)

- {

- std::list<FileListElem>::iterator it;

- std::string str;

-

- rOut.mSwapFeatures = mSwapFeatures;

- rOut.mStartFrameExt = mStartFrameExt;

- rOut.mEndFrameExt = mEndFrameExt;

- rOut.mTargetKind = mTargetKind;

- rOut.mDerivOrder = mDerivOrder;

- rOut.mDerivWinLengths = mDerivWinLengths;

-

- rOut.mpCvgFile = mpCvgFile;

- rOut.mpCmnPath = mpCmnPath;

- rOut.mpCmnMask = mpCmnMask;

- rOut.mpCvnPath = mpCvnPath;

- rOut.mpCvnMask = mpCvnMask;

-

- rOut.mInputQueue.clear();

-

- // go through all records and check the mask

- for (it=mInputQueue.begin(); it!= mInputQueue.end(); ++it) {

- if (pFilter == NULL

- || (ProcessMask(it->Logical(), pFilter, str) && (str == pValue))) {

- rOut.mInputQueue.push_back(*it);

- }

- }

-

- // set the queue position to the begining

- rOut.mInputQueueIterator = mInputQueue.end();

-

- rOut.mCurrentIndexFileName = "";

- rOut.mCurrentIndexFileDir = "";

- rOut.mCurrentIndexFileExt = "";

-

- mStream.close();

- mStream.clear();

-

- rOut.mpLastFileName = NULL;

- rOut.mLastFileName = "";

- rOut.mpLastCmnFile = NULL;

- rOut.mpLastCvnFile = NULL;

- rOut.mpLastCvgFile = NULL;

- rOut.mpCmn = NULL;

- rOut.mpCvn = NULL;

- rOut.mpCvg = NULL;

- rOut.mpA = NULL;

- rOut.mpB = NULL;

-

- }

-

-

- //***************************************************************************

- //***************************************************************************

- void

- FeatureRepository::

- HtkSelection(const char* pFilter, std::list< std::string >& rOut)

- {

- std::map< std::string, bool> aux_map;

- std::map< std::string, bool>::iterator map_it;

- std::list<FileListElem>::iterator it;

- std::string str;

-

- rOut.clear();

-

- if(pFilter != NULL) {

- // go through all records and check the mask

- for (it=mInputQueue.begin(); it!= mInputQueue.end(); ++it) {

- if (ProcessMask(it->Logical(), pFilter, str)) {

- aux_map[str] = true;

- }

- }

- } else {

- aux_map[std::string("default speaker")] = true;

- }

-

- for (map_it = aux_map.begin(); map_it != aux_map.end(); ++map_it) {

- rOut.push_back(map_it->first);

- }

- }

-

-

- //***************************************************************************

- //***************************************************************************

- int

- FeatureRepository::

- ParmKind2Str(unsigned parmKind, char *pOutString)

- {

- // :KLUDGE: Absolutely no idea what this is...

- if ((parmKind & 0x003F) >= sizeof(mpParmKindNames)/sizeof(mpParmKindNames[0]))

- return 0;

-

- strcpy(pOutString, mpParmKindNames[parmKind & 0x003F]);

-

- if (parmKind & PARAMKIND_E) strcat(pOutString, "_E");

- if (parmKind & PARAMKIND_N) strcat(pOutString, "_N");

- if (parmKind & PARAMKIND_D) strcat(pOutString, "_D");

- if (parmKind & PARAMKIND_A) strcat(pOutString, "_A");

- if (parmKind & PARAMKIND_C) strcat(pOutString, "_C");

- if (parmKind & PARAMKIND_Z) strcat(pOutString, "_Z");

- if (parmKind & PARAMKIND_K) strcat(pOutString, "_K");

- if (parmKind & PARAMKIND_0) strcat(pOutString, "_0");

- if (parmKind & PARAMKIND_V) strcat(pOutString, "_V");

- if (parmKind & PARAMKIND_T) strcat(pOutString, "_T");

-

- return 1;

- }

-

-

- // //***************************************************************************

- // //***************************************************************************

- // void

- // AddFileListToFeatureRepositories(

- // const char* pFileName,

- // const char* pFilter,

- // std::queue<FeatureRepository *> &featureRepositoryList)

- // {

- // IStkStream l_stream;

- // std::string file_name;

- // Tokenizer file_list(pFileName, ",");

- // Tokenizer::iterator p_file_name;

-

- // //:TODO: error if empty featureRepositoryList

- //

- // for (p_file_name = file_list.begin(); p_file_name != file_list.end(); ++p_file_name)

- // {

- // // get rid of initial and trailing blanks

- // Trim(*p_file_name);

-

- // // open file name

- // l_stream.open(p_file_name->c_str(), std::ios::in, pFilter);

- //

- // if (!l_stream.good()) {

- // //:TODO:

- // // Warning or error ... Why warning? -Lukas

- // throw std::runtime_error(std::string("Cannot not open list file ") +

- // *p_file_name);

- // }

-

- // // read all lines and parse them

- // for(;;)

- // {

- // l_stream >> file_name;

- // //:TODO: if(l_stream.badl()) Error()

- // // Reading after last token set the fail bit

- // if(l_stream.fail())

- // break;

- // // we can push_back a std::string as new FileListElem object

- // // is created using FileListElem(const std::string&) constructor

- // // and logical and physical names are correctly extracted

- // featureRepositoryList.front()->mInputQueue.push_back(file_name);

- //

- // //cycle in the featureRepositoryList

- // featureRepositoryList.push(featureRepositoryList.front());

- // featureRepositoryList.pop();

- // }

- // l_stream.close();

- // }

- // } // AddFileList(const std::string & rFileName)

-

-

- //***************************************************************************

- //***************************************************************************

- void

- FeatureRepository::

- Init(

- bool swap,

- int extLeft,

- int extRight,

- int targetKind,

- int derivOrder,

- int* pDerivWinLen,

- const char* pCmnPath,

- const char* pCmnMask,

- const char* pCvnPath,

- const char* pCvnMask,

- const char* pCvgFile)

- {

- mSwapFeatures = swap;

- mStartFrameExt = extLeft;

- mEndFrameExt = extRight;

- mTargetKind = targetKind;

- mDerivOrder = derivOrder;

- mDerivWinLengths = pDerivWinLen;

- mpCmnPath = pCmnPath;

- mpCmnMask = pCmnMask;

- mpCvnPath = pCvnPath;

- mpCvnMask = pCvnMask;

- mpCvgFile = pCvgFile;

- } // Init()

-

-

- //***************************************************************************

- //***************************************************************************

- void

- FeatureRepository::

- AddFile(const std::string & rFileName)

- {

- mInputQueue.push_back(rFileName);

- } // AddFile(const std::string & rFileName)

-

-

- //***************************************************************************

- //***************************************************************************

- void

- FeatureRepository::

- AddFileList(const char* pFileName, const char* pFilter)

- {

- IStkStream l_stream;

- std::string file_name;

- Tokenizer file_list(pFileName, ",");

- Tokenizer::iterator p_file_name;

-

- for (p_file_name = file_list.begin(); p_file_name != file_list.end(); ++p_file_name)

- {

- // get rid of spaces

- Trim(*p_file_name);

-

- // open the file

- l_stream.open(p_file_name->c_str(), std::ios::in, pFilter);

-

- if (!l_stream.good())

- {

- //:TODO:

- // Warning or error ... Why warning? -Lukas

- throw std::runtime_error(std::string("Cannot not open list file ") +

- *p_file_name);

- }

- // read all lines and parse them

- for(;;)

- {

- l_stream >> file_name;

- //:TODO: if(l_stream.badl()) Error()

- // Reading after last token set the fail bit

- if(l_stream.fail())

- break;

- // we can push_back a std::string as new FileListElem object

- // is created using FileListElem(const std::string&) constructor

- // and logical and physical names are correctly extracted

- mInputQueue.push_back(file_name);

- }

- l_stream.close();

- }

- } // AddFileList(const std::string & rFileName)

-

-

- //***************************************************************************

- //***************************************************************************

- void

- FeatureRepository::

- MoveNext()

- {

- assert (mInputQueueIterator != mInputQueue.end());

- mInputQueueIterator++;

- } // ReadFullMatrix(Matrix<BaseFloat>& rMatrix)

-

-

- //***************************************************************************

- //***************************************************************************

- bool

- FeatureRepository::

- ReadFullMatrix(Matrix<BaseFloat>& rMatrix)

- {

- // clear the matrix

- rMatrix.Destroy();

-

- // extract index file name

- if (!mCurrentIndexFileDir.empty())

- {

- char tmp_name[mCurrentIndexFileDir.length() +

- mCurrentIndexFileExt.length() +

- mInputQueueIterator->Physical().length()];

-

- MakeHtkFileName(tmp_name, mInputQueueIterator->Physical().c_str(),

- mCurrentIndexFileDir.c_str(), mCurrentIndexFileExt.c_str());

-

- mCurrentIndexFileName = tmp_name;

- }

- else

- mCurrentIndexFileName = "";

-

- //get the 3-letter suffix

- int pos_last_three_chars = mInputQueueIterator->Physical().size() - 3;

- if (pos_last_three_chars < 0) pos_last_three_chars = 0;

- //read the gzipped ascii features

- if (mInputQueueIterator->Physical().substr(pos_last_three_chars) == ".gz") {

- return ReadGzipAsciiFeatures(*mInputQueueIterator, rMatrix);

- }

-

- // read the matrix and return the result

- return ReadHTKFeatures(*mInputQueueIterator, rMatrix);

- } // ReadFullMatrix(Matrix<BaseFloat>& rMatrix)

-

-

-

- //***************************************************************************

- //***************************************************************************

- bool

- FeatureRepository::

- WriteFeatureMatrix(const Matrix<BaseFloat>& rMatrix, const std::string& filename, int targetKind, int samplePeriod)

- {

- FILE* fp = fopen(filename.c_str(),"w");

- if(NULL == fp) { Error(std::string("Cannot create file:") + filename); return false; }

-

- WriteHTKFeatures(fp, samplePeriod, targetKind, mSwapFeatures, const_cast<Matrix<BaseFloat>&>(rMatrix));

-

- fclose(fp);

-

- return true;

- }

-

-

- //***************************************************************************

- //***************************************************************************

- // private:

- int

- FeatureRepository::

- ReadHTKHeader()

- {

- // TODO

- // Change this... We should read from StkStream

- FILE* fp = mStream.fp();

-

- if (!fread(&mHeader.mNSamples, sizeof(INT_32), 1, fp)) return -1;

- if (!fread(&mHeader.mSamplePeriod, sizeof(INT_32), 1, fp)) return -1;

- if (!fread(&mHeader.mSampleSize, sizeof(INT_16), 1, fp)) return -1;

- if (!fread(&mHeader.mSampleKind, sizeof(UINT_16), 1, fp)) return -1;

-

- if (mSwapFeatures)

- {

- swap4(mHeader.mNSamples);

- swap4(mHeader.mSamplePeriod);

- swap2(mHeader.mSampleSize);

- swap2(mHeader.mSampleKind);

- }

-

- if (mHeader.mSamplePeriod < 0

- || mHeader.mSamplePeriod > 1000000

- || mHeader.mNSamples < 0

- || mHeader.mSampleSize < 0)

- {

- return -1;

- }

-

- return 0;

- }

-

-

- //***************************************************************************

- //***************************************************************************

- // private:

- int

- FeatureRepository::

- ReadHTKFeature(

- BaseFloat* pIn,

- size_t feaLen,

- bool decompress,

- BaseFloat* pScale,

- BaseFloat* pBias)

- {

- FILE* fp = mStream.fp();

-

- size_t i;

-

- if (decompress)

- {

- INT_16 s;

- // BaseFloat pScale = (xmax - xmin) / (2*32767);

- // BaseFloat pBias = (xmax + xmin) / 2;

-

- for (i = 0; i < feaLen; i++)

- {

- if (fread(&s, sizeof(INT_16), 1, fp) != 1)

- return -1;

-

- if (mSwapFeatures) swap2(s);

- pIn[i] = ((BaseFloat)s + pBias[i]) / pScale[i];

- }

-

- return 0;

- }

-

-#if !DOUBLEPRECISION

- if (fread(pIn, sizeof(FLOAT_32), feaLen, fp) != feaLen)

- return -1;

-

- if (mSwapFeatures)

- for (i = 0; i < feaLen; i++)

- swap4(pIn[i]);

-#else

- float f;

-

- for (i = 0; i < feaLen; i++)

- {

- if (fread(&f, sizeof(FLOAT_32), 1, fp) != 1)

- return -1;

-

- if (mSwapFeatures)

- swap4(f);

-

- pIn[i] = f;

- }

-#endif

- return 0;

- } // int ReadHTKFeature

-

-

-

- //***************************************************************************

- //***************************************************************************

-/* bool

- FeatureRepository::

- ReadHTKFeatures(const std::string& rFileName, Matrix<BaseFloat>& rFeatureMatrix)

- {

- std::string file_name(rFileName);

- std::string cmn_file_name;

- std::string cvn_file_name;

-

- int ext_left = mStartFrameExt;

- int ext_right = mEndFrameExt;

- int from_frame;

- int to_frame;

- int tot_frames;

- int trg_vec_size;

- int src_vec_size;

- int src_deriv_order;

- int lo_src_tgz_deriv_order;

- int i;

- int j;

- int k;

- int e;

- int coefs;

- int trg_E;

- int trg_0;

- int trg_N;

- int src_E;

- int src_0;

- int src_N;

- int comp;

- int coef_size;

- char* chptr;

-

-

-

- // read frame range definition if any ( physical_file.fea[s,e] )

- if ((chptr = strrchr(file_name.c_str(), '[')) == NULL ||

- ((i=0), sscanf(chptr, "[%d,%d]%n", &from_frame, &to_frame, &i),

- chptr[i] != '\0'))

- {

- chptr = NULL;

- }

-

- if (chptr != NULL)

- *chptr = '\0';

-

- // Experimental changes...

- // if ((strcmp(file_name.c_str(), "-"))

- // && (mpLastFileName != NULL)

- // && (!strcmp(mpLastFileName, file_name.c_str())))

- // {

- // mHeader = mLastHeader;

- // }

- // else

- // {

- // if (mpLastFileName)

- // {

- // //if (mpFp != stdin)

- // // fclose(mpFp);

- // mStream.close();

- //

- // free(mpLastFileName);

- // mpLastFileName = NULL;

- // }

-

- if ((file_name != "-" )

- && (!mLastFileName.empty())

- && (mLastFileName == file_name))

- {

- mHeader = mLastHeader;

- }

- else

- {

- if (!mLastFileName.empty())

- {

- mStream.close();

- mLastFileName = "";

- }

-

-

- // open the feature file

- mStream.open(file_name.c_str(), ios::binary);

- if (!mStream.good())

- {

- Error("Cannot open feature file: '%s'", file_name.c_str());

- }

-

-

- if (ReadHTKHeader())

- Error("Invalid HTK header in feature file: '%s'", file_name.c_str());

-

- if (mHeader.mSampleKind & PARAMKIND_C)

- {

- // File is in compressed form, scale and pBias vectors

- // are appended after HTK header.

-

- int coefs = mHeader.mSampleSize/sizeof(INT_16);

- mpA = (BaseFloat*) realloc(mpA, coefs * sizeof(BaseFloat));

- mpB = (BaseFloat*) realloc(mpB, coefs * sizeof(BaseFloat));

- if (mpA == NULL || mpB == NULL) Error("Insufficient memory");

-

- e = ReadHTKFeature(mpA, coefs, 0, 0, 0);

- e |= ReadHTKFeature(mpB, coefs, 0, 0, 0);

-

- if (e)

- Error("Cannot read feature file: '%s'", file_name.c_str());

-

- mHeader.mNSamples -= 2 * sizeof(FLOAT_32) / sizeof(INT_16);

- }

-

- // remember current settings

- mLastFileName = file_name;

- mLastHeader = mHeader;

- }

-

- if (chptr != NULL)

- *chptr = '[';

-

- if (chptr == NULL)

- { // Range [s,e] was not specified

- from_frame = 0;

- to_frame = mHeader.mNSamples-1;

- }

-

- src_deriv_order = PARAMKIND_T & mHeader.mSampleKind ? 3 :

- PARAMKIND_A & mHeader.mSampleKind ? 2 :

- PARAMKIND_D & mHeader.mSampleKind ? 1 : 0;

- src_E = (PARAMKIND_E & mHeader.mSampleKind) != 0;

- src_0 = (PARAMKIND_0 & mHeader.mSampleKind) != 0;

- src_N = ((PARAMKIND_N & mHeader.mSampleKind) != 0) * (src_E + src_0);

- comp = PARAMKIND_C & mHeader.mSampleKind;

-

- mHeader.mSampleKind &= ~PARAMKIND_C;

-

- if (mTargetKind == PARAMKIND_ANON)

- {

- mTargetKind = mHeader.mSampleKind;

- }

- else if ((mTargetKind & 077) == PARAMKIND_ANON)

- {

- mTargetKind &= ~077;

- mTargetKind |= mHeader.mSampleKind & 077;

- }

-

- trg_E = (PARAMKIND_E & mTargetKind) != 0;

- trg_0 = (PARAMKIND_0 & mTargetKind) != 0;

- trg_N =((PARAMKIND_N & mTargetKind) != 0) * (trg_E + trg_0);

-

- coef_size = comp ? sizeof(INT_16) : sizeof(FLOAT_32);

- coefs = (mHeader.mSampleSize/coef_size + src_N) /

- (src_deriv_order+1) - src_E - src_0;

- src_vec_size = (coefs + src_E + src_0) * (src_deriv_order+1) - src_N;

-

- //Is coefs dividable by 1 + number of derivatives specified in header

- if (src_vec_size * coef_size != mHeader.mSampleSize)

- {

- Error("Invalid HTK header in feature file: '%s'. "

- "mSampleSize do not match with parmKind", file_name.c_str());

- }

-

- if (mDerivOrder < 0)

- mDerivOrder = src_deriv_order;

-

-

- if ((!src_E && trg_E) || (!src_0 && trg_0) || (src_N && !trg_N) ||

- (trg_N && !trg_E && !trg_0) || (trg_N && !mDerivOrder) ||

- (src_N && !src_deriv_order && mDerivOrder) ||

- ((mHeader.mSampleKind & 077) != (mTargetKind & 077) &&

- (mHeader.mSampleKind & 077) != PARAMKIND_ANON))

- {

- char srcParmKind[64];

- char trgParmKind[64];

-

- ParmKind2Str(mHeader.mSampleKind, srcParmKind);

- ParmKind2Str(mTargetKind, trgParmKind);

- Error("Cannot convert %s to %s", srcParmKind, trgParmKind);

- }

-

- lo_src_tgz_deriv_order = LOWER_OF(src_deriv_order, mDerivOrder);

- trg_vec_size = (coefs + trg_E + trg_0) * (mDerivOrder+1) - trg_N;

-

- i = LOWER_OF(from_frame, mStartFrameExt);

- from_frame -= i;

- ext_left -= i;

-

- i = LOWER_OF(mHeader.mNSamples-to_frame-1, mEndFrameExt);

- to_frame += i;

- ext_right -= i;

-

- if (from_frame > to_frame || from_frame >= mHeader.mNSamples || to_frame< 0)

- Error("Invalid frame range for feature file: '%s'", file_name.c_str());

-

- tot_frames = to_frame - from_frame + 1 + ext_left + ext_right;

-

- // initialize matrix

- rFeatureMatrix.Init(tot_frames, trg_vec_size);

-

- // fill the matrix with features

- for (i = 0; i <= to_frame - from_frame; i++)

- {

- BaseFloat* A = mpA;

- BaseFloat* B = mpB;

- BaseFloat* mxPtr = rFeatureMatrix[i+ext_left];

-

- // seek to the desired position

- fseek(mStream.fp(),

- sizeof(HtkHeader) + (comp ? src_vec_size * 2 * sizeof(FLOAT_32) : 0)

- + (from_frame + i) * src_vec_size * coef_size,

- SEEK_SET);

-

- e = ReadHTKFeature(mxPtr, coefs, comp, A, B);

-

- mxPtr += coefs;

- A += coefs;

- B += coefs;

-

- if (src_0 && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);

- if (trg_0 && !trg_N) mxPtr++;

- if (src_E && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);

- if (trg_E && !trg_N) mxPtr++;

-

- for (j = 0; j < lo_src_tgz_deriv_order; j++)

- {

- e |= ReadHTKFeature(mxPtr, coefs, comp, A, B);

- mxPtr += coefs;

- A += coefs;

- B += coefs;

-

- if (src_0) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);

- if (trg_0) mxPtr++;

- if (src_E) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);

- if (trg_E) mxPtr++;

- }

-

- if (e)

- Error("Cannot read feature file: '%s' frame %d/%d", file_name.c_str(),

- i, to_frame - from_frame + 1);

- }

-

- // From now, coefs includes also trg_0 + trg_E !

- coefs += trg_0 + trg_E;

-

- // If extension of the matrix to the left or to the right is required,

- // perform it here

- for (i = 0; i < ext_left; i++)

- {

- memcpy(rFeatureMatrix[i],

- rFeatureMatrix[ext_left],

- (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat));

- }

-

- for (i = tot_frames - ext_right; i < tot_frames; i++)

- {

- memcpy(rFeatureMatrix[i],

- rFeatureMatrix[tot_frames - ext_right - 1],

- (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat));

- }

-

- // Sentence cepstral mean normalization

- if( (mpCmnPath == NULL)

- && !(PARAMKIND_Z & mHeader.mSampleKind)

- && (PARAMKIND_Z & mTargetKind))

- {

- // for each coefficient

- for(j=0; j < coefs; j++)

- {

- BaseFloat norm = 0.0;

- for(i=0; i < tot_frames; i++) // for each frame

- {

- norm += rFeatureMatrix[i][j - trg_N];

- //norm += fea_mx[i*trg_vec_size - trg_N + j];

- }

-

- norm /= tot_frames;

-

- for(i=0; i < tot_frames; i++) // for each frame

- rFeatureMatrix[i][j - trg_N] -= norm;

- //fea_mx[i*trg_vec_size - trg_N + j] -= norm;

- }

- }

-

- // Compute missing derivatives

- for (; src_deriv_order < mDerivOrder; src_deriv_order++)

- {

- int winLen = mDerivWinLengths[src_deriv_order];

- BaseFloat norm = 0.0;

-

- for (k = 1; k <= winLen; k++)

- {

- norm += 2 * k * k;

- }

-

- // for each frame

- for (i=0; i < tot_frames; i++)

- {

- // for each coefficient

- for (j=0; j < coefs; j++)

- {

- //BaseFloat* src = fea_mx + i*trg_vec_size + src_deriv_order*coefs - trg_N + j;

- BaseFloat* src = &rFeatureMatrix[i][src_deriv_order*coefs - trg_N + j];

-

- *(src + coefs) = 0.0;

-

- if (i < winLen || i >= tot_frames-winLen)

- { // boundaries need special treatment

- for (k = 1; k <= winLen; k++)

- {

- *(src+coefs) += k*(src[ LOWER_OF(tot_frames-1-i,k)*rFeatureMatrix.Stride()]

- -src[-LOWER_OF(i, k)*rFeatureMatrix.Stride()]);

- }

- }

- else

- { // otherwise use more efficient code

- for (k = 1; k <= winLen; k++)

- {

- *(src+coefs) += k*(src[ k * rFeatureMatrix.Stride()]

- -src[-k * rFeatureMatrix.Stride()]);

- }

- }

- *(src + coefs) /= norm;

- }

- }

- }

-

- mHeader.mNSamples = tot_frames;

- mHeader.mSampleSize = trg_vec_size * sizeof(FLOAT_32);

- mHeader.mSampleKind = mTargetKind & ~(PARAMKIND_D | PARAMKIND_A | PARAMKIND_T);

-

-

- ////////////////////////////////////////////////////////////////////////////

- /////////////// Cepstral mean and variance normalization ///////////////////

- ////////////////////////////////////////////////////////////////////////////

- //.........................................................................

- if (mpCmnPath != NULL

- && mpCmnMask != NULL)

- {

- // retrieve file name

- ProcessMask(file_name, mpCmnMask, cmn_file_name);

- // add the path correctly

- cmn_file_name.insert(0, "/");

- cmn_file_name.insert(0, mpCmnPath);

-

- // read the file

- ReadCepsNormFile(cmn_file_name.c_str(), &mpLastCmnFile, &mpCmn,

- mHeader.mSampleKind & ~PARAMKIND_Z, CNF_Mean, coefs);

-

- // recompute feature values

- for (i=0; i < tot_frames; i++)

- {

- for (j=trg_N; j < coefs; j++)

- {

- rFeatureMatrix[i][j - trg_N] -= mpCmn[j];

- }

- }

- }

-

- mHeader.mSampleKind |= mDerivOrder==3 ? PARAMKIND_D | PARAMKIND_A | PARAMKIND_T :

- mDerivOrder==2 ? PARAMKIND_D | PARAMKIND_A :

- mDerivOrder==1 ? PARAMKIND_D : 0;

-

- //.........................................................................

- if (mpCvnPath != NULL

- && mpCvnMask != NULL)

- {

- // retrieve file name

- ProcessMask(file_name, mpCvnMask, cvn_file_name);

- // add the path correctly

- cvn_file_name.insert(0, "/");

- cvn_file_name.insert(0, mpCvnPath);

-

- // read the file

- ReadCepsNormFile(cvn_file_name.c_str(), &mpLastCvnFile, &mpCvn,

- mHeader.mSampleKind, CNF_Variance, trg_vec_size);

-

- // recompute feature values

- for (i=0; i < tot_frames; i++)

- {

- for (j=trg_N; j < trg_vec_size; j++)

- {

- rFeatureMatrix[i][j - trg_N] *= mpCvn[j];

- }

- }

- }

-

- //.........................................................................

- // process the global covariance file

- if (mpCvgFile != NULL)

- {

- ReadCepsNormFile(mpCvgFile, &mpLastCvgFile, &mpCvg,

- -1, CNF_VarScale, trg_vec_size);

-

- // recompute feature values

- for (i=0; i < tot_frames; i++)

- {

- for (j=trg_N; j < trg_vec_size; j++)

- {

- rFeatureMatrix[i][j - trg_N] *= mpCvg[j];

- }

- }

- }

-

- return true;

- }

-*/

-

- //***************************************************************************

- //***************************************************************************

-

-

-

-

-

- //***************************************************************************

- //***************************************************************************

- bool

- FeatureRepository::

- ReadHTKFeatures(const FileListElem& rFileNameRecord,

- Matrix<BaseFloat>& rFeatureMatrix)

- {

- std::string file_name(rFileNameRecord.Physical());

- std::string cmn_file_name;

- std::string cvn_file_name;

-

- int ext_left = mStartFrameExt;

- int ext_right = mEndFrameExt;

- int from_frame;

- int to_frame;

- int tot_frames;

- int trg_vec_size;

- int src_vec_size;

- int src_deriv_order;

- int lo_src_tgz_deriv_order;

- int i;

- int j;

- int k;

- int e;

- int coefs;

- int trg_E;

- int trg_0;

- int trg_N;

- int src_E;

- int src_0;

- int src_N;

- int comp;

- int coef_size;

- char* chptr;

-

-

- TIMER_START(mTim);

-

- // read frame range definition if any ( physical_file.fea[s,e] )

- if ((chptr = strrchr((char*)file_name.c_str(), '[')) == NULL ||

- ((i=0), sscanf(chptr, "[%d,%d]%n", &from_frame, &to_frame, &i),

- chptr[i] != '\0'))

- {

- chptr = NULL;

- }

-

- if (chptr != NULL)

- *chptr = '\0';

-

-

- if ((file_name != "-" )

- && (!mLastFileName.empty())

- && (mLastFileName == file_name))

- {

- mHeader = mLastHeader;

- }

- else

- {

- if (!mLastFileName.empty())

- {

- mStream.close();

- mLastFileName = "";

- }

-

-

- // open the feature file

- mStream.open(file_name.c_str(), std::ios::binary);

- if (!mStream.good())

- {

- throw std::runtime_error(std::string("Cannot open feature file: '")

- + file_name.c_str() + "'");

- }

-

-

- if (ReadHTKHeader()) {

- throw std::runtime_error(std::string("Invalid HTK header in feature file: '")

- + file_name.c_str() + "'");

- }

-

- if (mHeader.mSampleKind & PARAMKIND_C)

- {

- // File is in compressed form, scale and pBias vectors

- // are appended after HTK header.

- coefs = mHeader.mSampleSize/sizeof(INT_16);

-

- mpA = (BaseFloat*) realloc(mpA, coefs * sizeof(BaseFloat));

- mpB = (BaseFloat*) realloc(mpB, coefs * sizeof(BaseFloat));

-

- if (mpA == NULL || mpB == NULL) {

- throw std::runtime_error("Insufficient memory");

- }

-

- e = ReadHTKFeature(mpA, coefs, 0, 0, 0);

- e |= ReadHTKFeature(mpB, coefs, 0, 0, 0);

-

- if (e) {

- throw std::runtime_error(std::string("Cannot read feature file: '")

- + file_name.c_str() + "'");

- }

-

- mHeader.mNSamples -= 2 * sizeof(FLOAT_32) / sizeof(INT_16);

- }

-

- // remember current settings

- mLastFileName = file_name;

- mLastHeader = mHeader;

- }

-

- if (chptr != NULL) {

- *chptr = '[';

- }

-

- if (chptr == NULL) {

- // Range [s,e] was not specified

- from_frame = 0;

- to_frame = mHeader.mNSamples-1;

- }

-

- src_deriv_order = PARAMKIND_T & mHeader.mSampleKind ? 3 :

- PARAMKIND_A & mHeader.mSampleKind ? 2 :

- PARAMKIND_D & mHeader.mSampleKind ? 1 : 0;

- src_E = (PARAMKIND_E & mHeader.mSampleKind) != 0;

- src_0 = (PARAMKIND_0 & mHeader.mSampleKind) != 0;

- src_N = ((PARAMKIND_N & mHeader.mSampleKind) != 0) * (src_E + src_0);

- comp = PARAMKIND_C & mHeader.mSampleKind;

-

- mHeader.mSampleKind &= ~PARAMKIND_C;

-

- if (mTargetKind == PARAMKIND_ANON)

- {

- mTargetKind = mHeader.mSampleKind;

- }

- else if ((mTargetKind & 077) == PARAMKIND_ANON)

- {

- mTargetKind &= ~077;

- mTargetKind |= mHeader.mSampleKind & 077;

- }

-

- trg_E = (PARAMKIND_E & mTargetKind) != 0;

- trg_0 = (PARAMKIND_0 & mTargetKind) != 0;

- trg_N =((PARAMKIND_N & mTargetKind) != 0) * (trg_E + trg_0);

-

- coef_size = comp ? sizeof(INT_16) : sizeof(FLOAT_32);

- coefs = (mHeader.mSampleSize/coef_size + src_N) /

- (src_deriv_order+1) - src_E - src_0;

- src_vec_size = (coefs + src_E + src_0) * (src_deriv_order+1) - src_N;

-

- //Is coefs dividable by 1 + number of derivatives specified in header

- if (src_vec_size * coef_size != mHeader.mSampleSize)

- {

- throw std::runtime_error(std::string("Invalid HTK header in feature file: '")

- + file_name + "' mSampleSize do not match with parmKind");

- }

-

- if (mDerivOrder < 0)

- mDerivOrder = src_deriv_order;

-

-

- if ((!src_E && trg_E) || (!src_0 && trg_0) || (src_N && !trg_N) ||

- (trg_N && !trg_E && !trg_0) || (trg_N && !mDerivOrder) ||

- (src_N && !src_deriv_order && mDerivOrder) ||

- ((mHeader.mSampleKind & 077) != (mTargetKind & 077) &&

- (mHeader.mSampleKind & 077) != PARAMKIND_ANON))

- {

- char srcParmKind[64];

- char trgParmKind[64];

- memset(srcParmKind,0,64);

- memset(trgParmKind,0,64);

-

- ParmKind2Str(mHeader.mSampleKind, srcParmKind);

- ParmKind2Str(mTargetKind, trgParmKind);

- throw std::runtime_error(std::string("Cannot convert ") + srcParmKind

- + " to " + trgParmKind);

- }

-

- lo_src_tgz_deriv_order = std::min(src_deriv_order, mDerivOrder);

- trg_vec_size = (coefs + trg_E + trg_0) * (mDerivOrder+1) - trg_N;

-

- i = std::min(from_frame, mStartFrameExt);

- from_frame -= i;

- ext_left -= i;

-

- i = std::min(mHeader.mNSamples-to_frame-1, mEndFrameExt);

- to_frame += i;

- ext_right -= i;

-

- if (from_frame > to_frame || from_frame >= mHeader.mNSamples || to_frame< 0)

- throw std::runtime_error(std::string("Invalid frame range for feature file: '")

- + file_name.c_str() + "'");

-

- tot_frames = to_frame - from_frame + 1 + ext_left + ext_right;

-

-

- TIMER_END(mTim,mTimeOpen);

-

-

- // initialize matrix

- rFeatureMatrix.Init(tot_frames, trg_vec_size, false);

-

- // fill the matrix with features

- for (i = 0; i <= to_frame - from_frame; i++)

- {

- BaseFloat* A = mpA;

- BaseFloat* B = mpB;

- BaseFloat* mxPtr = rFeatureMatrix.pRowData(i+ext_left);

-

- TIMER_START(mTim);

- // seek to the desired position

- fseek(mStream.fp(),

- sizeof(HtkHeader) + (comp ? src_vec_size * 2 * sizeof(FLOAT_32) : 0)

- + (from_frame + i) * src_vec_size * coef_size,

- SEEK_SET);

- TIMER_END(mTim,mTimeSeek);

-

- TIMER_START(mTim);

- // read

- e = ReadHTKFeature(mxPtr, coefs, comp, A, B);

- TIMER_END(mTim,mTimeRead);

-

- mxPtr += coefs;

- A += coefs;

- B += coefs;

-

- if (src_0 && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);

- if (trg_0 && !trg_N) mxPtr++;

- if (src_E && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);

- if (trg_E && !trg_N) mxPtr++;

-

- for (j = 0; j < lo_src_tgz_deriv_order; j++)

- {

- e |= ReadHTKFeature(mxPtr, coefs, comp, A, B);

- mxPtr += coefs;

- A += coefs;

- B += coefs;

-

- if (src_0) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);

- if (trg_0) mxPtr++;

- if (src_E) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);

- if (trg_E) mxPtr++;

- }

-

- if (e) {

- std::cout << mHeader.mNSamples << "\n";

- std::cout << 2 * sizeof(FLOAT_32) / sizeof(INT_16) << "\n";

- std::cout << "from" << from_frame << "to" << to_frame << "i" << i << "\n";

-

- std::ostringstream s;

- s << i << "/" << to_frame - from_frame + 1, s.str();

- throw std::runtime_error(std::string("Cannot read feature file: '")

- + file_name + "' frame " + s.str());

- }

- }

-

- // From now, coefs includes also trg_0 + trg_E !

- coefs += trg_0 + trg_E;

-

- // If extension of the matrix to the left or to the right is required,

- // perform it here

- for (i = 0; i < ext_left; i++)

- {

- memcpy(rFeatureMatrix.pRowData(i),

- rFeatureMatrix.pRowData(ext_left),

- (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat));

- }

-

- for (i = tot_frames - ext_right; i < tot_frames; i++)

- {

- memcpy(rFeatureMatrix.pRowData(i),

- rFeatureMatrix.pRowData(tot_frames - ext_right - 1),

- (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat));

- }

-

- // Sentence cepstral mean normalization

- if( (mpCmnPath == NULL)

- && !(PARAMKIND_Z & mHeader.mSampleKind)

- && (PARAMKIND_Z & mTargetKind))

- {

- // for each coefficient

- for(j=0; j < coefs; j++)

- {

- BaseFloat norm = 0.0;

- for(i=0; i < tot_frames; i++) // for each frame

- {

- norm += rFeatureMatrix[i][j - trg_N];

- //norm += fea_mx[i*trg_vec_size - trg_N + j];

- }

-

- norm /= tot_frames;

-

- for(i=0; i < tot_frames; i++) // for each frame

- rFeatureMatrix[i][j - trg_N] -= norm;

- //fea_mx[i*trg_vec_size - trg_N + j] -= norm;

- }

- }

-

- // Compute missing derivatives

- for (; src_deriv_order < mDerivOrder; src_deriv_order++)

- {

- int winLen = mDerivWinLengths[src_deriv_order];

- BaseFloat norm = 0.0;

-

- for (k = 1; k <= winLen; k++)

- {

- norm += 2 * k * k;

- }

-

- // for each frame

- for (i=0; i < tot_frames; i++)

- {

- // for each coefficient

- for (j=0; j < coefs; j++)

- {

- //BaseFloat* src = fea_mx + i*trg_vec_size + src_deriv_order*coefs - trg_N + j;

- BaseFloat* src = &rFeatureMatrix[i][src_deriv_order*coefs - trg_N + j];

-

- *(src + coefs) = 0.0;

-

- if (i < winLen || i >= tot_frames-winLen)

- { // boundaries need special treatment

- for (k = 1; k <= winLen; k++)

- {

- *(src+coefs) += k*(src[ std::min(tot_frames-1-i,k)*rFeatureMatrix.Stride()]

- -src[-std::min(i, k)*rFeatureMatrix.Stride()]);

- }

- }

- else

- { // otherwise use more efficient code

- for (k = 1; k <= winLen; k++)

- {

- *(src+coefs) += k*(src[ k * rFeatureMatrix.Stride()]

- -src[-k * rFeatureMatrix.Stride()]);

- }

- }

- *(src + coefs) /= norm;

- }

- }

- }

-

- mHeader.mNSamples = tot_frames;

- mHeader.mSampleSize = trg_vec_size * sizeof(FLOAT_32);

- mHeader.mSampleKind = mTargetKind & ~(PARAMKIND_D | PARAMKIND_A | PARAMKIND_T);

-

-

- TIMER_START(mTim);

- ////////////////////////////////////////////////////////////////////////////

- /////////////// Cepstral mean and variance normalization ///////////////////

- ////////////////////////////////////////////////////////////////////////////

- //.........................................................................

- if (mpCmnPath != NULL

- && mpCmnMask != NULL)

- {

- // retrieve file name

- ProcessMask(rFileNameRecord.Logical(), mpCmnMask, cmn_file_name);

- // add the path correctly

-

- if(cmn_file_name == "") {

- throw std::runtime_error("CMN Matching failed");

- }

-

- cmn_file_name.insert(0, "/");

- cmn_file_name.insert(0, mpCmnPath);

-

- // read the file

- ReadCepsNormFile(cmn_file_name.c_str(), &mpLastCmnFile, &mpCmn,

- mHeader.mSampleKind & ~PARAMKIND_Z, CNF_Mean, coefs);

-

- // recompute feature values

- for (i=0; i < tot_frames; i++)

- {

- for (j=trg_N; j < coefs; j++)

- {

- rFeatureMatrix[i][j - trg_N] -= mpCmn[j];

- }

- }

- }

-

- mHeader.mSampleKind |= mDerivOrder==3 ? PARAMKIND_D | PARAMKIND_A | PARAMKIND_T :

- mDerivOrder==2 ? PARAMKIND_D | PARAMKIND_A :

- mDerivOrder==1 ? PARAMKIND_D : 0;

-

- //.........................................................................

- if (mpCvnPath != NULL

- && mpCvnMask != NULL)

- {

- // retrieve file name

- ProcessMask(rFileNameRecord.Logical(), mpCvnMask, cvn_file_name);

- // add the path correctly

- cvn_file_name.insert(0, "/");

- cvn_file_name.insert(0, mpCvnPath);

-

- // read the file

- ReadCepsNormFile(cvn_file_name.c_str(), &mpLastCvnFile, &mpCvn,

- mHeader.mSampleKind, CNF_Variance, trg_vec_size);

-

- // recompute feature values

- for (i=0; i < tot_frames; i++)

- {

- for (j=trg_N; j < trg_vec_size; j++)

- {

- rFeatureMatrix[i][j - trg_N] *= mpCvn[j];

- }

- }

- }

-

- //.........................................................................

- // process the global covariance file

- if (mpCvgFile != NULL)

- {

- ReadCepsNormFile(mpCvgFile, &mpLastCvgFile, &mpCvg,

- -1, CNF_VarScale, trg_vec_size);

-

- // recompute feature values

- for (i=0; i < tot_frames; i++)

- {

- for (j=trg_N; j < trg_vec_size; j++)

- {

- rFeatureMatrix[i][j - trg_N] *= mpCvg[j];

- }

- }

- }

-

- TIMER_END(mTim,mTimeNormalize);

-

- return true;

- }

-

-

- //***************************************************************************

- //***************************************************************************

- int

- FeatureRepository::

- ReadParmKind(const char *str, bool checkBrackets)

- {

- unsigned int i;

- int parmKind =0;

- int slen = strlen(str);

-

- if (checkBrackets)

- {

- if (str[0] != '<' || str[slen-1] != '>') return -1;

- str++; slen -= 2;

- }

-

- for (; slen >= 0 && str[slen-2] == '_'; slen -= 2)

- {

- parmKind |= str[slen-1] == 'E' ? PARAMKIND_E :

- str[slen-1] == 'N' ? PARAMKIND_N :

- str[slen-1] == 'D' ? PARAMKIND_D :

- str[slen-1] == 'A' ? PARAMKIND_A :

- str[slen-1] == 'C' ? PARAMKIND_C :

- str[slen-1] == 'Z' ? PARAMKIND_Z :

- str[slen-1] == 'K' ? PARAMKIND_K :

- str[slen-1] == '0' ? PARAMKIND_0 :

- str[slen-1] == 'V' ? PARAMKIND_V :

- str[slen-1] == 'T' ? PARAMKIND_T : -1;

-

- if (parmKind == -1) return -1;

- }

-

- for (i = 0; i < sizeof(mpParmKindNames) / sizeof(char*); i++)

- {

- if (!strncmp(str, mpParmKindNames[i], slen))

- return parmKind | i;

- }

- return -1;

- }

-

-

-

-

- //***************************************************************************

- //***************************************************************************

- int

- FeatureRepository::

- WriteHTKHeader (FILE * pOutFp, HtkHeader header, bool swap)

- {

- int cc;

-

- if (swap) {

- swap4(header.mNSamples);

- swap4(header.mSamplePeriod);

- swap2(header.mSampleSize);

- swap2(header.mSampleKind);

- }

-

- fseek (pOutFp, 0L, SEEK_SET);

- cc = fwrite(&header, sizeof(HtkHeader), 1, pOutFp);

-

- if (swap) {

- swap4(header.mNSamples);

- swap4(header.mSamplePeriod);

- swap2(header.mSampleSize);

- swap2(header.mSampleKind);

- }

-

- return cc == 1 ? 0 : -1;

- }

-

-

- //***************************************************************************

- //***************************************************************************

- int

- FeatureRepository::

- WriteHTKFeature(

- FILE * pOutFp,

- FLOAT * pOut,

- size_t feaLen,

- bool swap,

- bool compress,

- FLOAT* pScale,

- FLOAT* pBias)

- {

- size_t i;

- size_t cc = 0;

-

-

- if (compress)

- {

- INT_16 s;

-

- for (i = 0; i < feaLen; i++)

- {

- s = pOut[i] * pScale[i] - pBias[i];

- if (swap)

- swap2(s);

- cc += fwrite(&s, sizeof(INT_16), 1, pOutFp);

- }

-

- } else {

- #if !DOUBLEPRECISION

- if (swap)

- for (i = 0; i < feaLen; i++)

- swap4(pOut[i]);

-

- cc = fwrite(pOut, sizeof(FLOAT_32), feaLen, pOutFp);

-

- if (swap)

- for (i = 0; i < feaLen; i++)

- swap4(pOut[i]);

- #else

- FLOAT_32 f;

-

- for (i = 0; i < feaLen; i++)

- {

- f = pOut[i];

- if (swap)

- swap4(f);

- cc += fwrite(&f, sizeof(FLOAT_32), 1, pOutFp);

- }

- #endif

- }

- return cc == feaLen ? 0 : -1;

- }

-

- //***************************************************************************

- //***************************************************************************

- int

- FeatureRepository::

- WriteHTKFeatures(

- FILE * pOutFp,

- FLOAT * pOut,

- int nCoeffs,

- int nSamples,

- int samplePeriod,

- int targetKind,

- bool swap)

- {

- HtkHeader header;

- int i, j;

- FLOAT *pScale = NULL;

- FLOAT *pBias = NULL;

-

- header.mNSamples = nSamples + ((targetKind & PARAMKIND_C) ? 2 * sizeof(FLOAT_32) / sizeof(INT_16) : 0);

- header.mSamplePeriod = samplePeriod;

- header.mSampleSize = nCoeffs * ((targetKind & PARAMKIND_C) ? sizeof(INT_16) : sizeof(FLOAT_32));;

- header.mSampleKind = targetKind;

-

- WriteHTKHeader (pOutFp, header, swap);

-

- if(targetKind & PARAMKIND_C) {

- pScale = (FLOAT*) malloc(nCoeffs * sizeof(FLOAT));

- pBias = (FLOAT*) malloc(nCoeffs * sizeof(FLOAT));

- if (pScale == NULL || pBias == NULL) Error("Insufficient memory");

-

- for(i = 0; i < nCoeffs; i++) {

- float xmin, xmax;

- xmin = xmax = pOut[i];

- for(j = 1; j < nSamples; j++) {

- if(pOut[j*nCoeffs+i] > xmax) xmax = pOut[j*nCoeffs+i];

- if(pOut[j*nCoeffs+i] < xmin) xmin = pOut[j*nCoeffs+i];

- }

- pScale[i] = (2*32767) / (xmax - xmin);

- pBias[i] = pScale[i] * (xmax + xmin) / 2;

-

-

- }

- if (WriteHTKFeature(pOutFp, pScale, nCoeffs, swap, false, 0, 0)

- || WriteHTKFeature(pOutFp, pBias, nCoeffs, swap, false, 0, 0)) {

- return -1;

- }

- }

- for(j = 0; j < nSamples; j++) {

- if (WriteHTKFeature(pOutFp, &pOut[j*nCoeffs], nCoeffs, swap, targetKind & PARAMKIND_C, pScale, pBias)) {

- return -1;

- }

- }

- return 0;

- }

-

-

- //***************************************************************************

- //***************************************************************************

- int

- FeatureRepository::

- WriteHTKFeatures(

- FILE * pOutFp,

- int samplePeriod,

- int targetKind,

- bool swap,

- Matrix<BaseFloat>& rFeatureMatrix)

- {

- HtkHeader header;

- size_t i, j;

- FLOAT *p_scale = NULL;

- FLOAT *p_bias = NULL;

- size_t n_samples = rFeatureMatrix.Rows();

- size_t n_coeffs = rFeatureMatrix.Cols();

-

- header.mNSamples = n_samples + ((targetKind & PARAMKIND_C) ? 2 * sizeof(FLOAT_32) / sizeof(INT_16) : 0);

- header.mSamplePeriod = samplePeriod;

- header.mSampleSize = n_coeffs * ((targetKind & PARAMKIND_C) ? sizeof(INT_16) : sizeof(FLOAT_32));;

- header.mSampleKind = targetKind;

-

- WriteHTKHeader (pOutFp, header, swap);

-

- if(targetKind & PARAMKIND_C) {

- p_scale = (FLOAT*) malloc(n_coeffs * sizeof(FLOAT));

- p_bias = (FLOAT*) malloc(n_coeffs * sizeof(FLOAT));

- if (p_scale == NULL || p_bias == NULL) Error("Insufficient memory");

-

- for(i = 0; i < n_coeffs; i++) {

- float xmin, xmax;

- xmin = xmax = rFeatureMatrix[0][i];

-

- for(j = 1; j < n_samples; j++) {

- if(rFeatureMatrix[j][i] > xmax) xmax = rFeatureMatrix[j][i];

- if(rFeatureMatrix[j][i] < xmin) xmin = rFeatureMatrix[j][i];

- }

-

- p_scale[i] = (2*32767) / (xmax - xmin);

- p_bias[i] = p_scale[i] * (xmax + xmin) / 2;

- }

-

- if (WriteHTKFeature(pOutFp, p_scale, n_coeffs, swap, false, 0, 0)

- || WriteHTKFeature(pOutFp, p_bias, n_coeffs, swap, false, 0, 0)) {

- return -1;

- }

- }

-

- for(j = 0; j < n_samples; j++) {

- if (WriteHTKFeature(pOutFp, rFeatureMatrix[j].pData(), n_coeffs, swap, targetKind & PARAMKIND_C, p_scale, p_bias)) {

- return -1;

- }

- }

-

- return 0;

- }

-

- //***************************************************************************

- //***************************************************************************

-

-

- bool

- FeatureRepository::

- ReadGzipAsciiFeatures(const FileListElem& rFileNameRecord, Matrix<BaseFloat>& rFeatureMatrix)

- {

- //build the command

- std::string cmd("gunzip -c "); cmd += rFileNameRecord.Physical();

-

- //define buffer

- const int buf_size=262144;

- char buf[buf_size];

- char vbuf[2*buf_size];

-

- TIMER_START(mTim);

- //open the pipe

- FILE* fp = popen(cmd.c_str(),"r");

- if(fp == NULL) {

- //2nd try...

- Warning(std::string("2nd try to open pipe: ")+cmd);

- sleep(5);

- fp = popen(cmd.c_str(),"r");

- if(fp == NULL) {

- KALDI_ERR << "Cannot open pipe: " << cmd;

- }

- }

- setvbuf(fp,vbuf,_IOFBF,2*buf_size);

- TIMER_END(mTim,mTimeOpen);

-

- //string will stay allocated across calls

- static std::string line; line.resize(0);

-

- //define matrix storage

- static int cols = 131072;

- std::list<std::vector<BaseFloat> > matrix(1);

- matrix.front().reserve(cols);

-

- //read all the lines to a vector

- int line_ctr=1;

- while(1) {

- TIMER_START(mTim);

- if(NULL == fgets(buf,buf_size,fp)) break;

- TIMER_END(mTim,mTimeRead);

-

- line += buf;

- if(*(line.rbegin()) == '\n' || feof(fp)) {

- //parse the line of numbers

- TIMER_START(mTim);

- const char* ptr = line.c_str();

- char* end;

- while(1) {

- //skip whitespace

- while(isspace(*ptr)) ptr++;

- if(*ptr == 0) break;

- //check that a number follows

- switch(*ptr) {

- case '0': case '1': case '2': case '3': case '4':

- case '5': case '6': case '7': case '8': case '9':

- case '.': case '+': case '-':

- break;

- default : KALDI_ERR << "A number was expected:" << ptr

- << " reading from" << cmd;

- exit(1);

- }

- //read a number

- BaseFloat val = strtof(ptr,&end); ptr=end;

- matrix.back().push_back(val);

- }

- TIMER_END(mTim,mTimeNormalize);

- //we have the line of numbers, insert empty row to matrix

- if(matrix.back().size() > 0 && !feof(fp)) {

- matrix.push_back(std::vector<BaseFloat>());

- matrix.back().reserve(matrix.front().size());

- }

- //dispose the current line

- line.resize(0);//but stay allocated...

- line_ctr++;

- }

- }

- if(matrix.back().size() == 0) matrix.pop_back();

-

- //get matrix dimensions

- int rows = matrix.size();

- /*int*/ cols = matrix.front().size();

-

- //define interators

- std::list<std::vector<BaseFloat> >::iterator it_r;

- std::vector<BaseFloat>::iterator it_c;

-

- //check that all lines have same size

- int i;

- for(i=0,it_r=matrix.begin(); it_r != matrix.end(); ++i,++it_r) {

- if(it_r->size() != cols) {

- KALDI_ERR << "All rows must have same dimension, 1st line cols: " << cols

- << ", " << i << "th line cols: " << it_r->size();

- }

- }

-

- //copy data to matrix

- TIMER_START(mTim);

- rFeatureMatrix.Init(rows,cols);

- int r,c;

- for(r=0,it_r=matrix.begin(); it_r!=matrix.end(); ++r,++it_r) {

- for(c=0,it_c=it_r->begin(); it_c!=it_r->end(); ++c,++it_c) {

- rFeatureMatrix(r,c) = *it_c;

- }

- }

- TIMER_END(mTim,mTimeSeek);

-

- //close the pipe

- if(pclose(fp) == -1) {

- KALDI_ERR << "Cannot close pipe: " << cmd;

- }

-

- return true;

- }

-

-

- //***************************************************************************

- //***************************************************************************

-

-} // namespace TNet

-//

-// C++ Interface: %{MODULE}

-//

-// Description:

-//

-//

-// Author: %{AUTHOR} <%{EMAIL}>, (C) %{YEAR}

-//

-// Copyright: See COPYING file that comes with this distribution

-//

-//

-

-#ifndef TNet_Features_h

-#define TNet_Features_h

-

-//*****************************************************************************

-//*****************************************************************************

-// Standard includes

-//

-#include <list>

-#include <queue>

-#include <string>

-

-

-//*****************************************************************************

-//*****************************************************************************

-// Specific includes

-//

-#include "Common.h"

-#include "Matrix.h"

-#include "StkStream.h"

-#include "Types.h"

-#include "Timer.h"

-

-

-

-// we need these for reading and writing

-#define UINT_16 unsigned short

-#define UINT_32 unsigned

-#define INT_16 short

-#define INT_32 int

-#define FLOAT_32 float

-#define DOUBLE_64 double

-

-

-#define PARAMKIND_WAVEFORM 0

-#define PARAMKIND_LPC 1

-#define PARAMKIND_LPREFC 2

-#define PARAMKIND_LPCEPSTRA 3

-#define PARAMKIND_LPDELCEP 4

-#define PARAMKIND_IREFC 5

-#define PARAMKIND_MFCC 6

-#define PARAMKIND_FBANK 7

-#define PARAMKIND_MELSPEC 8

-#define PARAMKIND_USER 9

-#define PARAMKIND_DISCRETE 10

-#define PARAMKIND_PLP 11

-#define PARAMKIND_ANON 12

-

-#define PARAMKIND_E 0000100 /// has energy

-#define PARAMKIND_N 0000200 /// absolute energy suppressed

-#define PARAMKIND_D 0000400 /// has delta coefficients

-#define PARAMKIND_A 0001000 /// has acceleration coefficients

-#define PARAMKIND_C 0002000 /// is compressed

-#define PARAMKIND_Z 0004000 /// has zero mean static coef.

-#define PARAMKIND_K 0010000 /// has CRC checksum

-#define PARAMKIND_0 0020000 /// has 0'th cepstral coef.

-#define PARAMKIND_V 0040000 /// has VQ codebook index

-#define PARAMKIND_T 0100000 /// has triple delta coefficients

-

-

-//*****************************************************************************

-//*****************************************************************************

-// Code ...

-//

-

-namespace TNet

-{

-

- /** **************************************************************************

- ** **************************************************************************

- */

- class FileListElem

- {

- private:

- std::string mLogical; ///< Logical file name representation

- std::string mPhysical; ///< Pysical file name representation

- float mWeight;

-

- public:

- FileListElem(const std::string & rFileName);

- ~FileListElem() {}

-

- const std::string &

- Logical() const { return mLogical; }

-

- const std::string &

- Physical() const { return mPhysical; }

-

- const float&

- Weight() const { return mWeight; }

- };

-

- /** *************************************************************************

- * @brief

- */

- class FeatureRepository

- {

- public:

- /**

- * @brief HTK parameter file header (see HTK manual)

- */

- struct HtkHeader

- {

- int mNSamples;

- int mSamplePeriod;

- short mSampleSize;

- short mSampleKind;

-

- HtkHeader()

- : mNSamples(0),mSamplePeriod(100000),mSampleSize(0),mSampleKind(12)

- { }

- };

-

-

- /**

- * @brief Extension of the HTK header

- */

- struct HtkHeaderExt

- {

- int mHeaderSize;

- int mVersion;

- int mSampSize;

- };

-

-

- /**

- * @brief Normalization file type

- */

- enum CNFileType

- {

- CNF_Mean,

- CNF_Variance,

- CNF_VarScale

- };

-

-

- static int

- ReadParmKind(const char *pStr, bool checkBrackets);

-

- static int

- ParmKind2Str(unsigned parmKind, char *pOutstr);

-

- static void

- ReadCepsNormFile(

- const char* pFileName,

- char** lastFile,

- BaseFloat** vecBuff,

- int sampleKind,

- CNFileType type,

- int coefs);

-

- static const char mpParmKindNames[13][16];

-

-

-

- //////////////////////////////////////////////////////////////////////////////

- // PUBLIC SECTION

- //////////////////////////////////////////////////////////////////////////////

- public:

- /// Iterates through the list of feature file records

- typedef std::list<FileListElem>::iterator ListIterator;

-

- // some params for loading features

- bool mSwapFeatures;

- int mStartFrameExt;

- int mEndFrameExt;

- int mTargetKind;

- int mDerivOrder;

- int* mDerivWinLengths;

- const char* mpCvgFile;

- //:TODO: get rid of these

- const char* mpCmnPath;

- const char* mpCmnMask;

- const char* mpCvnPath;

- const char* mpCvnMask;

-

- int mTrace;

-

-

- // Constructors and destructors

- /**

- * @brief Default constructor that creates an empty repository

- */

- FeatureRepository() : mDerivWinLengths(NULL), mpCvgFile(NULL),

- mpCmnPath(NULL), mpCmnMask(NULL), mpCvnPath(NULL), mpCvnMask(NULL),

- mTrace(0),

- mpLastFileName(NULL), mLastFileName(""), mpLastCmnFile (NULL),

- mpLastCvnFile (NULL), mpLastCvgFile (NULL), mpCmn(NULL),

- mpCvn(NULL), mpCvg(NULL), mpA(NULL), mpB(NULL),

- mTimeOpen(0), mTimeSeek(0), mTimeRead(0), mTimeNormalize(0)

- {

- mInputQueueIterator = mInputQueue.end();

- }

-

- /**

- * @brief Copy constructor which copies filled repository

- */

- FeatureRepository(const FeatureRepository& ori)

- : mDerivWinLengths(NULL), mpCvgFile(NULL),

- mpCmnPath(NULL), mpCmnMask(NULL), mpCvnPath(NULL), mpCvnMask(NULL),

- mTrace(0),

- mpLastFileName(NULL), mLastFileName(""), mpLastCmnFile (NULL),

- mpLastCvnFile (NULL), mpLastCvgFile (NULL), mpCmn(NULL),

- mpCvn(NULL), mpCvg(NULL), mpA(NULL), mpB(NULL),

- mTimeOpen(0), mTimeSeek(0), mTimeRead(0), mTimeNormalize(0)

- {

- //copy all the data from the input queue

- mInputQueue = ori.mInputQueue;

-

- //initialize like the original

- Init(

- ori.mSwapFeatures,

- ori.mStartFrameExt,

- ori.mEndFrameExt,

- ori.mTargetKind,

- ori.mDerivOrder,

- ori.mDerivWinLengths,

- ori.mpCmnPath,

- ori.mpCmnMask,

- ori.mpCvnPath,

- ori.mpCvnMask,

- ori.mpCvgFile);

-

- //set on the end

- mInputQueueIterator = mInputQueue.end();

- //copy default header values

- mHeader = ori.mHeader;

- }

-

-

- /**

- * @brief Destroys the repository

- */

- ~FeatureRepository()

- {

- if (NULL != mpA) {

- free(mpA);

- }

-

- if (NULL != mpB) {

- free(mpB);

- }

- //remove all entries

- mInputQueue.clear();

-

- if(mTrace&4) {

- std::cout << "[FeatureRepository -- open:" << mTimeOpen << "s seek:" << mTimeSeek << "s read:" << mTimeRead << "s normalize:" << mTimeNormalize << "s]\n";

- }

-

- }

-

-

- /**

- * @brief Initializes the object using the given parameters

- *

- * @param swap Boolean value specifies whether to swap bytes

- * when reading file or not.

- * @param extLeft Features read from file are extended with extLeft

- * initial frames. Normally, these frames are

- * repetitions of the first feature frame in file

- * (with its derivative, if derivatives are preset in

- * the file). However, if segment of feature frames

- * is extracted according to range specification, the

- * true feature frames from beyond the segment boundary

- * are used, wherever it is possible. Note that value

- * of extLeft can be also negative. In such case

- * corresponding number of initial frames is discarded.

- * @param extRight The paramerer is complementary to parameter extLeft

- * and has obvious meaning. (Controls extensions over

- * the last frame, last frame from file is repeated

- * only if necessary).

- * @param targetKind The parameters is used to check whether

- * pHeader->mSampleKind match to requited targetKind

- * and to control suppression of 0'th cepstral or

- * energy coefficients accorging to modifiers _E, _0,

- * and _N. Modifiers _D, _A and _T are ignored;

- * Computation of derivatives is controled by parameters

- * derivOrder and derivWinLen. Value PARAMKIND_ANON

- * ensures that function do not result in targetKind

- * mismatch error and cause no _E or _0 suppression.

- * @param derivOrder Final features will be augmented with their

- * derivatives up to 'derivOrder' order. If 'derivOrder'

- * is negative value, no new derivatives are appended

- * and derivatives that already present in feature file

- * are preserved. Straight features are considered

- * to be of zero order. If some derivatives are already

- * present in feature file, these are not computed

- * again, only higher order derivatives are appended

- * if required. Note, that HTK feature file cannot

- * contain higher order derivatives (e.g. double delta)

- * without containing lower ones (e.g. delta).

- * Derivative present in feature file that are of

- * higher order than is required are discarded.

- * Derivatives are computed in the final stage from

- * (extracted segment of) feature frames possibly

- * extended by repeated frames. Derivatives are

- * computed using the same formula that is employed

- * also by HTK tools. Lengths of windows used for

- * computation of derivatives are passed in parameter

- * derivWinLen. To compute derivatives for frames close

- * to boundaries, frames before the first and after the

- * last frame (of the extracted segment) are considered

- * to be (yet another) repetitions of the first and the

- * last frame, respectively. If the segment of frames

- * is extracted according to range specification and

- * parameters extLeft and extLeft are set to zero, the

- * first and the last frames of the segment are

- * considered to be repeated, eventough the true feature

- * frames from beyond the segment boundary can be

- * available in the file. Therefore, segment extracted

- * from features that were before augmented with

- * derivatives will differ

- * from the same segment augmented with derivatives by

- * this function. Difference will be of course only on

- * boundaries and only in derivatives. This "incorrect"

- * behavior was chosen to fully simulate behavior of

- * HTK tools. To obtain more correct computation of

- * derivatives, use parameters extLeft and extRight,

- * which correctly extend segment with the true frames

- * (if possible) and in resulting feature matrix ignore

- * first extLeft and last extRight frames. For this

- * purpose, both extLeft and extRight should be set to

- * sum of all values in the array derivWinLen.

- * @param pDerivWinLen Array of size derivOrder specifying lengths of

- * windows used for computation of derivatives.

- * Individual values represents one side context

- * used in the computation. The each window length is

- * therefore twice the value from array plus one.

- * Value at index zero specify window length for first

- * order derivatives (delta), higher indices

- * corresponds to higher order derivatives.

- * @param pCmnPath Cepstral mean normalization path

- * @param pCmnMask Cepstral mean normalization mask

- * @param pCvnPath Cepstral variance normalization path

- * @param pCvnMask Cepstral variance normalization mask

- * @param pCvgFile Global variance file to be parsed

- *

- * The given parameters are necessary for propper feature extraction

- */

- void

- Init(

- bool swap,

- int extLeft,

- int extRight,

- int targetKind,

- int derivOrder,

- int* pDerivWinLen,

- const char* pCmnPath,

- const char* pCmnMask,

- const char* pCvnPath,

- const char* pCvnMask,

- const char* pCvgFile);

-

-

- void Trace(int trace)

- { mTrace = trace; }

-

- /**

- * @brief Returns a refference to the current file header

- */

- const HtkHeader&

- CurrentHeader() const

- { return mHeader; }

-

- /**

- * @brief Returns a refference to the current file header

- */

- const HtkHeaderExt&

- CurrentHeaderExt() const

- { return mHeaderExt; }

-

- /**

- * @brief Returns the current file details

- *

- * @return Refference to a class @c FileListElem

- *

- * Logical and physical file names are stored in @c FileListElem class

- */

- const std::list<FileListElem>::iterator&

- pCurrentRecord() const

- { return mInputQueueIterator; }

-

-

- /**

- * @brief Returns the following file details

- *

- * @return Refference to a class @c FileListElem

- *

- * Logical and physical file names are stored in @c FileListElem class

- */

- const std::list<FileListElem>::iterator&

- pFollowingRecord() const

- { return mInputQueueIterator; }

-

-

- void

- Rewind()

- { mInputQueueIterator = mInputQueue.begin(); }

-

-

- /**

- * @brief Adds a single feature file to the repository

- * @param rFileName file to read features from

- */

- void

- AddFile(const std::string & rFileName);

-

-

- /**

- * @brief Adds a list of feature files to the repository

- * @param rFileName feature list file to read from

- */

- void

- AddFileList(const char* pFileName, const char* pFilter = "");

-

-

- const FileListElem&

- Current() const

- { return *mInputQueueIterator; }

-

-

- /**

- * @brief Moves to the next record

- */

- void

- MoveNext();

-

- /**

- * @brief Reads full feature matrix from a feature file

- * @param rMatrix matrix to be created and filled with read data

- * @return number of successfully read feature vectors

- */

- bool

- ReadFullMatrix(Matrix<BaseFloat>& rMatrix);

-

- bool

- WriteFeatureMatrix(const Matrix<BaseFloat>& rMatrix, const std::string& filename, int targetKind, int samplePeriod);

-

- size_t

- QueueSize() const {return mInputQueue.size(); }

-

- /**

- * @brief Reads feature vectors from a feature file

- * @param rMatrix matrix to be (only!) filled with read data.

- * @return number of successfully read feature vectors

- *

- * The function tries to fill @c pMatrix with feature vectors comming from

- * the current stream. If there are less vectors left in the stream,

- * they are used and true number of successfuly read vectors is returned.

- */

- int

- ReadPartialMatrix(Matrix<BaseFloat>& rMatrix);

-

- /**

- * @brief Filters the records of this repository based on HTK logical name

- * masking. If pFilter equals to NULL, all source repository entries are

- * coppied to rOut repository.

- *

- * @param pFilter HTK mask that defines the filter

- * @param pValue Filter value

- * @param rOut Reference to the new FeatureRepository which will be filled

- * with the matching records

- */

- void

- HtkFilter(const char* pFilter, const char* pValue, FeatureRepository& rOut);

-

-

- /**

- * @brief Filters the records of this repository based on HTK logical name

- * masking and returns list of unique names. If pFilter equals to NULL,

- * single name "default" is returned.

- *

- * @param pFilter HTK mask that defines the filter

- * @param rOut Reference to the list of results (std::list< std::string >)

- */

- void

- HtkSelection(const char* pFilter, std::list< std::string >& rOut);

-

-

- /**

- * @brief Returns true if there are no feature files left on input

- */

- bool

- EndOfList() const

- { return mInputQueueIterator == mInputQueue.end(); }

-

- const std::string&

- CurrentIndexFileName() const

- { return mCurrentIndexFileName; }

-

- friend

- void

- AddFileListToFeatureRepositories(

- const char* pFileName,

- const char* pFilter,

- std::queue<FeatureRepository *> &featureRepositoryList);

-

-

-////////////////////////////////////////////////////////////////////////////////

-// PRIVATE SECTION

-////////////////////////////////////////////////////////////////////////////////

- private:

- /// List (queue) of input feature files

- std::list<FileListElem> mInputQueue;

- std::list<FileListElem>::iterator mInputQueueIterator;

-

- std::string mCurrentIndexFileName;

- std::string mCurrentIndexFileDir;

- std::string mCurrentIndexFileExt;

-

- /// current stream

- IStkStream mStream;

-

- // stores feature file's HTK header

- HtkHeader mHeader;

- HtkHeaderExt mHeaderExt;

-

-

- // this group of variables serve for working withthe same physical

- // file name more than once

- char* mpLastFileName;

- std::string mLastFileName;

- char* mpLastCmnFile;

- char* mpLastCvnFile;

- char* mpLastCvgFile;

- BaseFloat* mpCmn;

- BaseFloat* mpCvn;

- BaseFloat* mpCvg;

- HtkHeader mLastHeader;

- BaseFloat* mpA;

- BaseFloat* mpB;

-

-

-

- Timer mTim;

- double mTimeOpen;

- double mTimeSeek;

- double mTimeRead;

- double mTimeNormalize;

-

-

- // Reads HTK feature file header

- int

- ReadHTKHeader();

-

- int

- ReadHTKFeature(BaseFloat* pIn,

- size_t feaLen,

- bool decompress,

- BaseFloat* pScale,

- BaseFloat* pBias);

-

-

- bool

- ReadHTKFeatures(const std::string& rFileName, Matrix<BaseFloat>& rFeatureMatrix);

-

- bool

- ReadHTKFeatures(const FileListElem& rFileNameRecord, Matrix<BaseFloat>& rFeatureMatrix);

-

-

- int

- WriteHTKHeader (FILE* fp_out, HtkHeader header, bool swap);

-

- int

- WriteHTKFeature (FILE* fp_out, FLOAT *out, size_t fea_len, bool swap, bool compress, FLOAT* pScale, FLOAT* pBias);

-

- int

- WriteHTKFeatures(FILE* pOutFp, FLOAT * pOut, int nCoeffs, int nSamples, int samplePeriod, int targetKind, bool swap);

-

- int

- WriteHTKFeatures(

- FILE * pOutFp,

- int samplePeriod,

- int targetKind,

- bool swap,

- Matrix<BaseFloat>& rFeatureMatrix

- );

-

- bool

- ReadGzipAsciiFeatures(const FileListElem& rFileNameRecord, Matrix<BaseFloat>& rFeatureMatrix);

-

- }; // class FeatureStream

-

-} //namespace TNet

-

-#endif // TNet_Features_h

-#include "Labels.h"

-#include "Timer.h"

-#include "Error.h"

-#include <cstdio>

-#include <sstream>

-

-

-namespace TNet {

-

-

- ////////////////////////////////////////////////////////////////////////

- // Class LabelRepository::

- void

- LabelRepository::

- Init(const char* pLabelMlfFile, const char* pOutputLabelMapFile, const char* pLabelDir, const char* pLabelExt) {

- InitMap(pLabelMlfFile, pOutputLabelMapFile, pLabelDir, pLabelExt);

- }

- void

- LabelRepository::

- InitExt(const char* pLabelMlfFile, const char* fmt, const char *arg, const char* pLabelDir, const char* pLabelExt) {

- if (strcmp(fmt, "map") == 0)

- {

- InitMap(pLabelMlfFile, arg, pLabelDir, pLabelExt);

- mlf_fmt = MAP;

- }

- else if (strcmp(fmt, "raw") == 0)

- {

- InitRaw(pLabelMlfFile, arg, pLabelDir, pLabelExt);

- mlf_fmt = RAW;

- }

- }

-

- void

- LabelRepository::

- InitMap(const char* pLabelMlfFile, const char* pOutputLabelMapFile, const char* pLabelDir, const char* pLabelExt)

- {

- assert(NULL != pLabelMlfFile);

- assert(NULL != pOutputLabelMapFile);

-

- // initialize the label streams

- delete mpLabelStream; //if NULL, does nothing

- delete _mpLabelStream;

- _mpLabelStream = new std::ifstream(pLabelMlfFile);

- mpLabelStream = new IMlfStream(*_mpLabelStream);

-

- // Label stream is initialized, just test it

- if(!mpLabelStream->good())

- Error(std::string("Cannot open Label MLF file: ")+pLabelMlfFile);

-

- // Index the labels (good for randomized file lists)

- Timer tim; tim.Start();

- mpLabelStream->Index();

- tim.End(); mIndexTime += tim.Val();

-

- // Read the state-label to state-id map

- ReadOutputLabelMap(pOutputLabelMapFile);

-

- // Store the label dir/ext

- mpLabelDir = pLabelDir;

- mpLabelExt = pLabelExt;

- }

-

- void

- LabelRepository::

- InitRaw(const char* pLabelMlfFile, const char *arg, const char* pLabelDir, const char* pLabelExt)

- {

- assert(NULL != pLabelMlfFile);

- std::istringstream iss(arg);

- size_t dim;

- iss >> dim;

- if (iss.fail() || dim <= 0)

- PError("[lab] malformed dimension specification");

- raw_dim = dim;

- // initialize the label streams

- delete mpLabelStream; //if NULL, does nothing

- delete _mpLabelStream;

- _mpLabelStream = new std::ifstream(pLabelMlfFile);

- mpLabelStream = new IMlfStream(*_mpLabelStream);

-

- // Label stream is initialized, just test it

- if(!mpLabelStream->good())

- Error(std::string("Cannot open Label MLF file: ")+pLabelMlfFile);

-

- // Index the labels (good for randomized file lists)

- Timer tim; tim.Start();

- mpLabelStream->Index();

- tim.End(); mIndexTime += tim.Val();

-

- // Read the state-label to state-id map

- //ReadOutputLabelMap(pOutputLabelMapFile);

-

- // Store the label dir/ext

- mpLabelDir = pLabelDir;

- mpLabelExt = pLabelExt;

- }

-

-

- void

- LabelRepository::

- GenDesiredMatrix(BfMatrix& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical, bool has_vad)

- {

- //timer

- Timer tim; tim.Start();

-

- //Get the MLF stream reference...

- IMlfStream& mLabelStream = *mpLabelStream;

- //Build the file name of the label

- MakeHtkFileName(mpLabelFile, pFeatureLogical, mpLabelDir, mpLabelExt);

-

- //Find block in MLF file

- mLabelStream.Open(mpLabelFile);

- if(!mLabelStream.good()) {

- Error(std::string("Cannot open label MLF record: ") + mpLabelFile);

- }

-

-

- //resize the matrix

- if(nFrames < 1) {

- KALDI_ERR << "Number of frames:" << nFrames << " is lower than 1!!!\n"

- << pFeatureLogical;

- }

- int label_map_size = mLabelMap.size();

- rDesired.Init(nFrames, label_map_size + (has_vad ? 2 : 0), true); //true: Zero()

-

- //aux variables

- std::string line, state;

- unsigned long long beg, end;

- size_t state_index;

- size_t trunc_frames = 0;

- TagToIdMap::iterator it;

- int vad_state;

-

- //parse the label file

- while(!mLabelStream.eof()) {

- std::getline(mLabelStream, line);

- if(line == "") continue; //skip newlines/comments from MLF

- if(line[0] == '#') continue;

-

- std::istringstream& iss = mGenDesiredMatrixStream;

- iss.clear();

- iss.str(line);

-

- //parse the line

- //begin

- iss >> std::ws >> beg;

- if(iss.fail()) {

- KALDI_ERR << "Cannot parse column 1 (begin)\n"

- << "line: " << line << "\n"

- << "file: " << mpLabelFile << "\n";

- }

- //end

- iss >> std::ws >> end;

- if(iss.fail()) {

- KALDI_ERR << "Cannot parse column 2 (end)\n"

- << "line: " << line << "\n"

- << "file: " << mpLabelFile << "\n";

- }

- //state tag

- iss >> std::ws >> state;

- if(iss.fail()) {

- KALDI_ERR << "Cannot parse column 3 (state_tag)\n"

- << "line: " << line << "\n"

- << "file: " << mpLabelFile << "\n";

- }

-

- if (has_vad) /* an additional column for vad */

- {

- iss >> std::ws >> vad_state;

- if(iss.fail()) {

- KALDI_ERR << "Cannot parse column 4 (vad_state)\n"

- << "line: " << line << "\n"

- << "file: " << mpLabelFile << "\n";

- }

- }

-

- //fprintf(stderr, "Parsed: %lld %lld %s\n", beg, end, state.c_str());

-

- //divide beg/end by sourceRate and round up to get interval of frames

- beg = (beg+sourceRate/2)/sourceRate;

- end = (end+sourceRate/2)/sourceRate;

- //beg = (int)round(beg / (double)sourceRate);

- //end = (int)round(end / (double)sourceRate);

-

- //find the state id

- it = mLabelMap.find(state);

- if(mLabelMap.end() == it) {

- Error(std::string("Unknown state tag: '") + state + "' file:'" + mpLabelFile);

- }

- state_index = it->second;

-

- // Fill the desired matrix

- for(unsigned long long frame=beg; frame<end; frame++) {

- //don't write after matrix... (possible longer transcript than feature file)

- if(frame >= (int)rDesired.Rows()) { trunc_frames++; continue; }

-

- //check the next frame is empty:

- if(0.0 != rDesired[frame].Sum()) {

- //ERROR!!!

- //find out what was previously filled!!!

- BaseFloat max = rDesired[frame].Max();

- int idx = -1;

- for(int i=0; i<(int)rDesired[frame].Dim(); i++) {

- if(rDesired[frame][i] == max) idx = i;

- }

- for(it=mLabelMap.begin(); it!=mLabelMap.end(); ++it) {

- if((int)it->second == idx) break;

- }

- std::string state_prev = "error";

- if(it != mLabelMap.end()) {

- state_prev = it->first;

- }

- //print the error message

- std::ostringstream os;

- os << "Frame already assigned to other state, "

- << " file: " << mpLabelFile

- << " frame: " << frame

- << " nframes: " << nFrames

- << " sum: " << rDesired[frame].Sum()

- << " previously assigned to: " << state_prev << "(" << idx << ")"

- << " now should be assigned to: " << state << "(" << state_index << ")"

- << "\n";

- Error(os.str());

- }

-

- //fill the row

- rDesired[(size_t)frame][state_index] = 1.0f;

- if (has_vad)

- rDesired[(size_t)frame][label_map_size + !!vad_state] = 1.0f;

- }

- }

-

- mLabelStream.Close();

-

- //check the desired matrix (rows sum up to 1.0)

- for(size_t i=0; i<rDesired.Rows(); ++i) {

- float desired_row_sum = rDesired[i].Sum();

- if(!desired_row_sum == 1.0) {

- std::ostringstream os;

- os << "Desired vector sum isn't 1.0, "

- << " file: " << mpLabelFile

- << " row: " << i

- << " nframes: " << nFrames

- << " content: " << rDesired[i]

- << " sum: " << desired_row_sum << "\n";

- Error(os.str());

- }

- }

-

- //warning when truncating many frames

- if(trunc_frames > 10) {

- std::ostringstream os;

- os << "Truncated frames: " << trunc_frames

- << " Check sourcerate in features and validity of labels\n";

- Warning(os.str());

- }

-

- //timer

- tim.End(); mGenDesiredMatrixTime += tim.Val();

- }

-

-

-

- void

- LabelRepository::

- ReadOutputLabelMap(const char* file)

- {

- assert(mLabelMap.size() == 0);

- int i = 0;

- std::string state_tag;

- std::ifstream in(file);

- if(!in.good())

- Error(std::string("Cannot open OutputLabelMapFile: ")+file);

-

- in >> std::ws;

- while(!in.eof()) {

- in >> state_tag;

- in >> std::ws;

- assert(mLabelMap.find(state_tag) == mLabelMap.end());

- mLabelMap[state_tag] = i++;

- }

-

- in.close();

- assert(mLabelMap.size() > 0);

- }

-

-

- void

- LabelRepository::

- GenDesiredMatrixExt(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical) {

- switch (mlf_fmt)

- {

- case MAP: GenDesiredMatrixExtMap(rDesired, nFrames, sourceRate, pFeatureLogical);

- break;

- case RAW: GenDesiredMatrixExtRaw(rDesired, nFrames, sourceRate, pFeatureLogical);

- break;

- default: assert(0);

- }

- }

-

- void

- LabelRepository::

- GenDesiredMatrixExtMap(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical)

- {

- //timer

- Timer tim; tim.Start();

-

- //Get the MLF stream reference...

- IMlfStream& mLabelStream = *mpLabelStream;

- //Build the file name of the label

- MakeHtkFileName(mpLabelFile, pFeatureLogical, mpLabelDir, mpLabelExt);

-

- //Find block in MLF file

- mLabelStream.Open(mpLabelFile);

- if(!mLabelStream.good()) {

- Error(std::string("Cannot open label MLF record: ") + mpLabelFile);

- }

-

-

- //resize the matrix

- if(nFrames < 1) {

- KALDI_ERR << "Number of frames:" << nFrames << " is lower than 1!!!\n"

- << pFeatureLogical;

- }

-

- size_t prev = rDesired.size();

- rDesired.resize(prev + 1, BfMatrix()); /* state + vad */

- int label_map_size = mLabelMap.size();

- rDesired[prev].Init(nFrames, 1, true); //true: Zero()

-

- //aux variables

- std::string line, state;

- unsigned long long beg, end;

- size_t state_index;

- size_t trunc_frames = 0;

- TagToIdMap::iterator it;

-

- //parse the label file

- while(!mLabelStream.eof()) {

- std::getline(mLabelStream, line);

- if(line == "") continue; //skip newlines/comments from MLF

- if(line[0] == '#') continue;

-

- std::istringstream& iss = mGenDesiredMatrixStream;

- iss.clear();

- iss.str(line);

-

- //parse the line

- //begin

- iss >> std::ws >> beg;

- if(iss.fail()) {

- KALDI_ERR << "Cannot parse column 1 (begin)\n"

- << "line: " << line << "\n"

- << "file: " << mpLabelFile << "\n";

- }

- //end

- iss >> std::ws >> end;

- if(iss.fail()) {

- KALDI_ERR << "Cannot parse column 2 (end)\n"

- << "line: " << line << "\n"

- << "file: " << mpLabelFile << "\n";

- }

- //state tag

- iss >> std::ws >> state;

- if(iss.fail()) {

- KALDI_ERR << "Cannot parse column 3 (state_tag)\n"

- << "line: " << line << "\n"

- << "file: " << mpLabelFile << "\n";

- }

-

-

- //fprintf(stderr, "Parsed: %lld %lld %s\n", beg, end, state.c_str());

-

- //divide beg/end by sourceRate and round up to get interval of frames

- beg = (beg+sourceRate/2)/sourceRate;

- if (end == (unsigned long long)-1)

- end = rDesired[prev].Rows();

- else

- end = (end+sourceRate/2)/sourceRate;

- //beg = (int)round(beg / (double)sourceRate);

- //end = (int)round(end / (double)sourceRate);

-

- //find the state id

- it = mLabelMap.find(state);

- if(mLabelMap.end() == it) {

- Error(std::string("Unknown state tag: '") + state + "' file:'" + mpLabelFile);

- }

- state_index = it->second;

-

- // Fill the desired matrix

- for(unsigned long long frame=beg; frame<end; frame++) {

- //don't write after matrix... (possible longer transcript than feature file)

- if(frame >= (int)rDesired[prev].Rows()) { trunc_frames++; continue; }

-

- //check the next frame is empty:

- if(0.0 != rDesired[prev][frame][0]) {

- //ERROR!!!

- //find out what was previously filled!!!

- /*

- BaseFloat max = rDesired[prev][frame].Max();

- int idx = -1;

- for(int i=0; i<(int)rDesired[prev][frame].Dim(); i++) {

- if(rDesired[prev][frame][i] == max) idx = i;

- }

- */

- BaseFloat max = rDesired[prev][frame][0];

- int idx = round(max);

- for(it=mLabelMap.begin(); it!=mLabelMap.end(); ++it) {

- if((int)it->second == idx) break;

- }

- std::string state_prev = "error";

- if(it != mLabelMap.end()) {

- state_prev = it->first;

- }

- //print the error message

- std::ostringstream os;

- os << "Frame already assigned to other state, "

- << " file: " << mpLabelFile

- << " frame: " << frame

- << " nframes: " << nFrames

- << " sum: " << max

- << " previously assigned to: " << state_prev << "(" << idx << ")"

- << " now should be assigned to: " << state << "(" << state_index << ")"

- << "\n";

- Error(os.str());

- }

-

- //fill the row

- //rDesired[prev][(size_t)frame][state_index] = 1.0f;

- rDesired[prev][(size_t)frame][0] = state_index;

- }

- }

-

- mLabelStream.Close();

- /*

- //check the desired matrix (rows sum up to 1.0)

- for(size_t i=0; i<rDesired[prev].Rows(); ++i) {

- float desired_row_sum = rDesired[prev][i].Sum();

- if(!desired_row_sum == 1.0) {

- std::ostringstream os;

- os << "Desired vector sum isn't 1.0, "

- << " file: " << mpLabelFile

- << " row: " << i

- << " nframes: " << nFrames

- << " content: " << rDesired[prev][i]

- << " sum: " << desired_row_sum << "\n";

- Error(os.str());

- }

- }

- */

-

- //warning when truncating many frames

- if(trunc_frames > 10) {

- std::ostringstream os;

- os << "Truncated frames: " << trunc_frames

- << " Check sourcerate in features and validity of labels\n";

- Warning(os.str());

- }

-

- //timer

- tim.End(); mGenDesiredMatrixTime += tim.Val();

- }

-

- void

- LabelRepository::

- GenDesiredMatrixExtRaw(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical)

- {

- //timer

- Timer tim; tim.Start();

-

- //Get the MLF stream reference...

- IMlfStream& mLabelStream = *mpLabelStream;

- //Build the file name of the label

- MakeHtkFileName(mpLabelFile, pFeatureLogical, mpLabelDir, mpLabelExt);

-

- //Find block in MLF file

- mLabelStream.Open(mpLabelFile);

- if(!mLabelStream.good()) {

- Error(std::string("Cannot open label MLF record: ") + mpLabelFile);

- }

-

-

- //resize the matrix

- if(nFrames < 1) {

- KALDI_ERR << "Number of frames:" << nFrames << " is lower than 1!!!\n"

- << pFeatureLogical;

- }

-

- size_t prev = rDesired.size();

- rDesired.resize(prev + 1, BfMatrix()); /* state + vad */

- rDesired[prev].Init(nFrames, raw_dim, true); //true: Zero()

-

- //aux variables

- std::string line, state;

- unsigned long long beg, end;

- size_t trunc_frames = 0;

- Vector<BaseFloat> raw;

- raw.Init(raw_dim);

-

- //parse the label file

- while(!mLabelStream.eof()) {

- std::getline(mLabelStream, line);

- if(line == "") continue; //skip newlines/comments from MLF

- if(line[0] == '#') continue;

-

- std::istringstream& iss = mGenDesiredMatrixStream;

- iss.clear();

- iss.str(line);

-

- //parse the line

- //begin

- iss >> std::ws >> beg;

- if(iss.fail()) {

- KALDI_ERR << "Cannot parse column 1 (begin)\n"

- << "line: " << line << "\n"

- << "file: " << mpLabelFile << "\n";

- }

- //end

- iss >> std::ws >> end;

- if(iss.fail()) {

- KALDI_ERR << "Cannot parse column 2 (end)\n"

- << "line: " << line << "\n"

- << "file: " << mpLabelFile << "\n";

- }

-

- for (size_t i = 0; i < raw_dim; i++)

- {

- if (iss.eof())

- PError("[label] insufficient columns for the label: %s", mpLabelFile);

- iss >> raw[i];

- if (iss.fail())

- PError("[label] cannot parse raw value for the label: %s", mpLabelFile);

- }

- /*

- for (size_t i = 0; i < raw_dim; i++)

- fprintf(stderr, "%.3f", raw[i]);

- fprintf(stderr, "\n");

- */

- //divide beg/end by sourceRate and round up to get interval of frames

- beg = (beg+sourceRate/2)/sourceRate;

- if (end == (unsigned long long)-1)

- end = rDesired[prev].Rows();

- else

- end = (end+sourceRate/2)/sourceRate;

- //printf("end:%lld\n", end);

- //beg = (int)round(beg / (double)sourceRate);

- //end = (int)round(end / (double)sourceRate);

-

- // Fill the desired matrix

- for(unsigned long long frame=beg; frame<end; frame++) {

- //don't write after matrix... (possible longer transcript than feature file)

- if(frame >= (int)rDesired[prev].Rows()) { trunc_frames++; continue; }

-

- //check the next frame is empty:

- if(0.0 != rDesired[prev][frame][0]) {

- //ERROR!!!

- //find out what was previously filled!!!

- /*

- BaseFloat max = rDesired[prev][frame].Max();

- int idx = -1;

- for(int i=0; i<(int)rDesired[prev][frame].Dim(); i++) {

- if(rDesired[prev][frame][i] == max) idx = i;

- }

- */

- BaseFloat max = rDesired[prev][frame][0];

- //print the error message

- std::ostringstream os;

- os << "Frame already assigned to other state, "

- << " file: " << mpLabelFile

- << " frame: " << frame

- << " nframes: " << nFrames

- << " sum: " << max

- << "\n";

- Error(os.str());

- }

-

- //fill the row

- //rDesired[prev][(size_t)frame][state_index] = 1.0f;

- rDesired[prev][(size_t)frame].Copy(raw);

- }

- }

-

- mLabelStream.Close();

- /*

- //check the desired matrix (rows sum up to 1.0)

- for(size_t i=0; i<rDesired[prev].Rows(); ++i) {

- float desired_row_sum = rDesired[prev][i].Sum();

- if(!desired_row_sum == 1.0) {

- std::ostringstream os;

- os << "Desired vector sum isn't 1.0, "

- << " file: " << mpLabelFile

- << " row: " << i

- << " nframes: " << nFrames

- << " content: " << rDesired[prev][i]

- << " sum: " << desired_row_sum << "\n";

- Error(os.str());

- }

- }

- */

-

- //warning when truncating many frames

- if(trunc_frames > 10) {

- std::ostringstream os;

- os << "Truncated frames: " << trunc_frames

- << " Check sourcerate in features and validity of labels\n";

- Warning(os.str());

- }

-

- //timer

- tim.End(); mGenDesiredMatrixTime += tim.Val();

- }

-

-}//namespace

-#ifndef _LABELS_H_

-#define _LABELS_H_

-

-

-#include "Matrix.h"

-#include "MlfStream.h"

-#include "Features.h"

-

-#include <map>

-#include <iostream>

-

-namespace TNet {

-

-

- class FeaCatPool;

-

- /**

- * Desired matrix generation object,

- * supports background-reading and caching, however can be

- * used in foreground as well by GenDesiredMatrix()

- */

- class LabelRepository

- {

- typedef std::map<std::string,size_t> TagToIdMap;

-

- public:

- enum MFormat {

- MAP,

- RAW

- };

-

- LabelRepository()

- : _mpLabelStream(NULL), mpLabelStream(NULL), mpLabelDir(NULL), mpLabelExt(NULL), mGenDesiredMatrixTime(0), mIndexTime(0), mTrace(0)

- { }

-

- ~LabelRepository()

- {

- if(mTrace&4) {

- std::cout << "[LabelRepository -- indexing:" << mIndexTime << "s"

- " genDesiredMatrix:" << mGenDesiredMatrixTime << "s]" << std::endl;

- }

- delete mpLabelStream;

- delete _mpLabelStream;

- }

-

- /// Initialize the LabelRepository

- void Init(const char* pLabelMlfFile, const char* pOutputLabelMapFile, const char* pLabelDir, const char* pLabelExt);

- void InitExt(const char* pLabelMlfFile, const char* fmt, const char* arg, const char* pLabelDir, const char* pLabelExt);

- void InitMap(const char* pLabelMlfFile, const char* pOutputLabelMapFile, const char* pLabelDir, const char* pLabelExt);

- void InitRaw(const char* pLabelMlfFile,const char* arg, const char* pLabelDir, const char* pLabelExt);

-

- /// Set trace level

- void Trace(int trace)

- { mTrace = trace; }

-

- /// Get desired matrix from labels

- void GenDesiredMatrix(BfMatrix& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical, bool has_vad = false);

-

- void GenDesiredMatrixExt(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical);

- void GenDesiredMatrixExtMap(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical);

- void GenDesiredMatrixExtRaw(std::vector<BfMatrix>& rDesired, size_t nFrames, size_t sourceRate, const char* pFeatureLogical);

- size_t getWidth() { return mLabelMap.size(); }

- MFormat getFormat() { return mlf_fmt; }

- private:

- /// Prepare the state-label to state-id map

- void ReadOutputLabelMap(const char* file);

-

- private:

- // Streams and state-map

- std::ifstream* _mpLabelStream; ///< Helper stream for Label stream

- IMlfStream* mpLabelStream; ///< Label stream

- std::istringstream mGenDesiredMatrixStream; ///< Label file parsing stream

-

- const char* mpLabelDir; ///< Label dir in MLF

- const char* mpLabelExt; ///< Label ext in MLF

- char mpLabelFile[4096]; ///< Buffer for filenames in MLF

-

- TagToIdMap mLabelMap; ///< Map of state tags to net output indices

-

- double mGenDesiredMatrixTime;

- float mIndexTime;

-

- int mTrace;

- MFormat mlf_fmt;

- size_t raw_dim;

- };

-

-}//namespace

-

-#endif

-

-include ../tnet.mk

-

-INCLUDE = -I.

-

-all: $(OBJ_DIR)/libKaldiLib.a

-

-$(OBJ_DIR)/libKaldiLib.a: $(OBJ)

- $(AR) ruv $@ $(OBJ)

- $(RANLIB) $@

-

-$(OBJ_DIR)/%.o : %.cc

- $(CXX) -o $@ -c $< $(CFLAGS) $(CXXFLAGS) $(INCLUDE)

-

-

-

-.PHONY: clean doc depend

-clean:

- rm -f $(OBJ_DIR)/*.o $(OBJ_DIR)/*.a

-

-doc:

- doxygen ../../doc/doxyfile_TNetLib

-

-depend:

- $(CXX) -M $(CXXFLAGS) *.cc $(INCLUDE) > .depend.mk

-

--include .depend.mk

-

-#ifndef TNet_MathAux_h

-#define TNet_MathAux_h

-

-#include <cmath>

-

-

-#if !defined(SQR)

-# define SQR(x) ((x) * (x))

-#endif

-

-

-#if !defined(LOG_0)

-# define LOG_0 (-1.0e10)

-#endif

-

-#if !defined(LOG_MIN)

-# define LOG_MIN (0.5 * LOG_0)

-#endif

-

-

-#ifndef DBL_EPSILON

-#define DBL_EPSILON 2.2204460492503131e-16

-#endif

-

-

-#ifndef M_PI

-# define M_PI 3.1415926535897932384626433832795

-#endif

-

-#define M_LOG_2PI 1.8378770664093454835606594728112

-

-

-#if DOUBLEPRECISION

-# define FLOAT double

-# define EPSILON DBL_EPSILON

-# define FLOAT_FMT "%lg"

-# define swapFLOAT swap8

-# define _ABS fabs

-# define _COS cos

-# define _EXP exp

-# define _LOG log

-# define _SQRT sqrt

-#else

-# define FLOAT float

-# define EPSILON FLT_EPSILON

-# define FLOAT_FMT "%g"

-# define swapFLOAT swap4

-# define _ABS fabsf

-# define _COS cosf

-# define _EXP expf

-# define _LOG logf

-# define _SQRT sqrtf

-#endif

-

-namespace TNet

-{

- inline float frand(){ // random between 0 and 1.

- return (float(rand()) + 1.0f) / (float(RAND_MAX)+2.0f);

- }

- inline float gauss_rand(){

- return _SQRT( -2.0f * _LOG(frand()) ) * _COS(2.0f*float(M_PI)*frand());

- }

-

- static const double gMinLogDiff = log(DBL_EPSILON);

-

- //***************************************************************************

- //***************************************************************************

- inline double

- LogAdd(double x, double y)

- {

- double diff;

-

- if (x < y) {

- diff = x - y;

- x = y;

- } else {

- diff = y - x;

- }

-

- double res;

- if (x >= LOG_MIN) {

- if (diff >= gMinLogDiff) {

- res = x + log(1.0 + exp(diff));

- } else {

- res = x;

- }

- } else {

- res = LOG_0;

- }

- return res;

- }

-

-

- //***************************************************************************

- //***************************************************************************

- inline double

- LogSub(double x, double y) // returns exp(x) - exp(y). Throws exception if y>=x.

- {

-

- if(y >= x){

- if(y==x) return LOG_0;

- else throw std::runtime_error("LogSub: cannot subtract a larger from a smaller number.");

- }

-

- double diff = y - x; // Will be negative.

-

- double res = x + log(1.0 - exp(diff));

-

- if(res != res) // test for res==NaN.. could happen if diff ~0.0, so 1.0-exp(diff) == 0.0 to machine precision.

- res = LOG_0;

- return res;

- }

-

-} // namespace TNet

-

-

-#endif

-/**

- * @file Matrix.cc

- *

- * Implementation of specialized Matrix template methods

- */

-

-

-#include "Matrix.h"

-

-#if defined(HAVE_CLAPACK)

-#include "CLAPACK-3.1.1.1/INCLUDE/f2c.h"

-extern "C" {

-#include "CLAPACK-3.1.1.1/INCLUDE/clapack.h"

-}

-// These are some stupid clapack things that we want to get rid of

-#ifdef min

-#undef min

-#endif

-

-#ifdef max

-#undef max

-#endif

-

-#endif

-

-

-

-

-namespace TNet

-{

- //***************************************************************************

- //***************************************************************************

-#ifdef HAVE_ATLAS

- //***************************************************************************

- //***************************************************************************

- template<>

- Matrix<float> &

- Matrix<float>::

- Invert(float *LogDet, float *DetSign, bool inverse_needed)

- {

- assert(Rows() == Cols());

-

-#if defined(HAVE_CLAPACK)

- integer* pivot = new integer[mMRows];

- integer M = Rows();

- integer N = Cols();

- integer LDA = mStride;

- integer result;

- integer l_work = std::max<integer>(1, N);

- float* p_work = new float[l_work];

-

- sgetrf_(&M, &N, mpData, &LDA, pivot, &result);

- const int pivot_offset=1;

-#else

- int* pivot = new int[mMRows];

- int result = clapack_sgetrf(CblasColMajor, Rows(), Cols(), mpData, mStride, pivot);

- const int pivot_offset=0;

-#endif

- assert(result >= 0 && "Call to CLAPACK sgetrf_ or ATLAS clapack_sgetrf called with wrong arguments");

- if(result != 0) {

- Error("Matrix is singular");

- }

- if(DetSign!=NULL){ *DetSign=1.0; for(size_t i=0;i<mMRows;i++) if(pivot[i]!=(int)i+pivot_offset) *DetSign *= -1.0; }

- if(LogDet!=NULL||DetSign!=NULL){ // Compute log determinant...

- assert(mMRows==mMCols); // Can't take determinant of non-square matrix.

- *LogDet = 0.0; float prod = 1.0;

- for(size_t i=0;i<mMRows;i++){

- prod *= (*this)(i,i);

- if(i==mMRows-1 || fabs(prod)<1.0e-10 || fabs(prod)>1.0e+10){

- if(LogDet!=NULL) *LogDet += log(fabs(prod));

- if(DetSign!=NULL) *DetSign *= (prod>0?1.0:-1.0);

- prod=1.0;

- }

- }

- }

-#if defined(HAVE_CLAPACK)

- if(inverse_needed) sgetri_(&M, mpData, &LDA, pivot, p_work, &l_work, &result);

- delete [] pivot;

-#else

- if(inverse_needed) result = clapack_sgetri(CblasColMajor, Rows(), mpData, mStride, pivot);

- delete [] pivot;

-#endif

- assert(result == 0 && "Call to CLAPACK sgetri_ or ATLAS clapack_sgetri called with wrong arguments");

- return *this;

- }

-

-

- //***************************************************************************

- //***************************************************************************

- template<>

- Matrix<double> &

- Matrix<double>::

- Invert(double *LogDet, double *DetSign, bool inverse_needed)

- {

- assert(Rows() == Cols());

-

-#if defined(HAVE_CLAPACK)

- integer* pivot = new integer[mMRows];

- integer M = Rows();

- integer N = Cols();

- integer LDA = mStride;

- integer result;

- integer l_work = std::max<integer>(1, N);

- double* p_work = new double[l_work];

-

- dgetrf_(&M, &N, mpData, &LDA, pivot, &result);

- const int pivot_offset=1;

-#else

- int* pivot = new int[mMRows];

- int result = clapack_dgetrf(CblasColMajor, Rows(), Cols(), mpData, mStride, pivot);

- const int pivot_offset=0;

-#endif

- assert(result >= 0 && "Call to CLAPACK dgetrf_ or ATLAS clapack_dgetrf called with wrong arguments");

- if(result != 0) {

- Error("Matrix is singular");

- }

- if(DetSign!=NULL){ *DetSign=1.0; for(size_t i=0;i<mMRows;i++) if(pivot[i]!=(int)i+pivot_offset) *DetSign *= -1.0; }

- if(LogDet!=NULL||DetSign!=NULL){ // Compute log determinant...

- assert(mMRows==mMCols); // Can't take determinant of non-square matrix.

- *LogDet = 0.0; double prod = 1.0;

- for(size_t i=0;i<mMRows;i++){

- prod *= (*this)(i,i);

- if(i==mMRows-1 || fabs(prod)<1.0e-10 || fabs(prod)>1.0e+10){

- if(LogDet!=NULL) *LogDet += log(fabs(prod));

- if(DetSign!=NULL) *DetSign *= (prod>0?1.0:-1.0);

- prod=1.0;

- }

- }

- }

-#if defined(HAVE_CLAPACK)

- if(inverse_needed) dgetri_(&M, mpData, &LDA, pivot, p_work, &l_work, &result);

- delete [] pivot;

-#else

- if(inverse_needed) result = clapack_dgetri(CblasColMajor, Rows(), mpData, mStride, pivot);

- delete [] pivot;

-#endif

- assert(result == 0 && "Call to CLAPACK dgetri_ or ATLAS clapack_dgetri called with wrong arguments");

- return *this;

- }

-

- template<>

- Matrix<float> &

- Matrix<float>::

- BlasGer(const float alpha, const Vector<float>& rA, const Vector<float>& rB)

- {

- assert(rA.Dim() == mMRows && rB.Dim() == mMCols);

- cblas_sger(CblasRowMajor, rA.Dim(), rB.Dim(), alpha, rA.pData(), 1, rB.pData(), 1, mpData, mStride);

- return *this;

- }

-

- template<>

- Matrix<double> &

- Matrix<double>::

- BlasGer(const double alpha, const Vector<double>& rA, const Vector<double>& rB)

- {

- assert(rA.Dim() == mMRows && rB.Dim() == mMCols);

- cblas_dger(CblasRowMajor, rA.Dim(), rB.Dim(), alpha, rA.pData(), 1, rB.pData(), 1, mpData, mStride);

- return *this;

- }

-

- template<>

- Matrix<float>&

- Matrix<float>::

- BlasGemm(const float alpha,

- const Matrix<float>& rA, MatrixTrasposeType transA,

- const Matrix<float>& rB, MatrixTrasposeType transB,

- const float beta)

- {

- assert((transA == NO_TRANS && transB == NO_TRANS && rA.Cols() == rB.Rows() && rA.Rows() == Rows() && rB.Cols() == Cols())

- || (transA == TRANS && transB == NO_TRANS && rA.Rows() == rB.Rows() && rA.Cols() == Rows() && rB.Cols() == Cols())

- || (transA == NO_TRANS && transB == TRANS && rA.Cols() == rB.Cols() && rA.Rows() == Rows() && rB.Rows() == Cols())

- || (transA == TRANS && transB == TRANS && rA.Rows() == rB.Cols() && rA.Cols() == Rows() && rB.Rows() == Cols()));

-

- cblas_sgemm(CblasRowMajor, static_cast<CBLAS_TRANSPOSE>(transA), static_cast<CBLAS_TRANSPOSE>(transB),

- Rows(), Cols(), transA == NO_TRANS ? rA.Cols() : rA.Rows(),

- alpha, rA.mpData, rA.mStride, rB.mpData, rB.mStride,

- beta, mpData, mStride);

- return *this;

- }

-

- template<>

- Matrix<double>&

- Matrix<double>::

- BlasGemm(const double alpha,

- const Matrix<double>& rA, MatrixTrasposeType transA,

- const Matrix<double>& rB, MatrixTrasposeType transB,

- const double beta)

- {

- assert((transA == NO_TRANS && transB == NO_TRANS && rA.Cols() == rB.Rows() && rA.Rows() == Rows() && rB.Cols() == Cols())

- || (transA == TRANS && transB == NO_TRANS && rA.Rows() == rB.Rows() && rA.Cols() == Rows() && rB.Cols() == Cols())

- || (transA == NO_TRANS && transB == TRANS && rA.Cols() == rB.Cols() && rA.Rows() == Rows() && rB.Rows() == Cols())

- || (transA == TRANS && transB == TRANS && rA.Rows() == rB.Cols() && rA.Cols() == Rows() && rB.Rows() == Cols()));

-

- cblas_dgemm(CblasRowMajor, static_cast<CBLAS_TRANSPOSE>(transA), static_cast<CBLAS_TRANSPOSE>(transB),

- Rows(), Cols(), transA == NO_TRANS ? rA.Cols() : rA.Rows(),

- alpha, rA.mpData, rA.mStride, rB.mpData, rB.mStride,

- beta, mpData, mStride);

- return *this;

- }

-

- template<>

- Matrix<float>&

- Matrix<float>::

- Axpy(const float alpha,

- const Matrix<float>& rA, MatrixTrasposeType transA){

- int aStride = (int)rA.mStride, stride = mStride;

- float *adata=rA.mpData, *data=mpData;

- if(transA == NO_TRANS){

- assert(rA.Rows()==Rows() && rA.Cols()==Cols());

- for(size_t row=0;row<mMRows;row++,adata+=aStride,data+=stride)

- cblas_saxpy(mMCols, alpha, adata, 1, data, 1);

- } else {

- assert(rA.Cols()==Rows() && rA.Rows()==Cols());

- for(size_t row=0;row<mMRows;row++,adata++,data+=stride)

- cblas_saxpy(mMCols, alpha, adata, aStride, data, 1);

- }

- return *this;

- }

-

- template<>

- Matrix<double>&

- Matrix<double>::

- Axpy(const double alpha,

- const Matrix<double>& rA, MatrixTrasposeType transA){

- int aStride = (int)rA.mStride, stride = mStride;

- double *adata=rA.mpData, *data=mpData;

- if(transA == NO_TRANS){

- assert(rA.Rows()==Rows() && rA.Cols()==Cols());

- for(size_t row=0;row<mMRows;row++,adata+=aStride,data+=stride)

- cblas_daxpy(mMCols, alpha, adata, 1, data, 1);

- } else {

- assert(rA.Cols()==Rows() && rA.Rows()==Cols());

- for(size_t row=0;row<mMRows;row++,adata++,data+=stride)

- cblas_daxpy(mMCols, alpha, adata, aStride, data, 1);

- }

- return *this;

- }

-

- template <> //non-member but friend!

- double TraceOfProduct(const Matrix<double> &A, const Matrix<double> &B){ // tr(A B), equivalent to sum of each element of A times same element in B'

- size_t aStride = A.mStride, bStride = B.mStride;

- assert(A.Rows()==B.Cols() && A.Cols()==B.Rows());

- double ans = 0.0;

- double *adata=A.mpData, *bdata=B.mpData;

- size_t arows=A.Rows(), acols=A.Cols();

- for(size_t row=0;row<arows;row++,adata+=aStride,bdata++)

- ans += cblas_ddot(acols, adata, 1, bdata, bStride);

- return ans;

- }

-

- template <> //non-member but friend!

- double TraceOfProductT(const Matrix<double> &A, const Matrix<double> &B){ // tr(A B), equivalent to sum of each element of A times same element in B'

- size_t aStride = A.mStride, bStride = B.mStride;

- assert(A.Rows()==B.Rows() && A.Cols()==B.Cols());

- double ans = 0.0;

- double *adata=A.mpData, *bdata=B.mpData;

- size_t arows=A.Rows(), acols=A.Cols();

- for(size_t row=0;row<arows;row++,adata+=aStride,bdata+=bStride)

- ans += cblas_ddot(acols, adata, 1, bdata, 1);

- return ans;

- }

-

-

- template <> //non-member but friend!

- float TraceOfProduct(const Matrix<float> &A, const Matrix<float> &B){ // tr(A B), equivalent to sum of each element of A times same element in B'

- size_t aStride = A.mStride, bStride = B.mStride;

- assert(A.Rows()==B.Cols() && A.Cols()==B.Rows());

- float ans = 0.0;

- float *adata=A.mpData, *bdata=B.mpData;

- size_t arows=A.Rows(), acols=A.Cols();

- for(size_t row=0;row<arows;row++,adata+=aStride,bdata++)

- ans += cblas_sdot(acols, adata, 1, bdata, bStride);

- return ans;

- }

-

- template <> //non-member but friend!

- float TraceOfProductT(const Matrix<float> &A, const Matrix<float> &B){ // tr(A B), equivalent to sum of each element of A times same element in B'

- size_t aStride = A.mStride, bStride = B.mStride;

- assert(A.Rows()==B.Rows() && A.Cols()==B.Cols());

- float ans = 0.0;

- float *adata=A.mpData, *bdata=B.mpData;

- size_t arows=A.Rows(), acols=A.Cols();

- for(size_t row=0;row<arows;row++,adata+=aStride,bdata+=bStride)

- ans += cblas_sdot(acols, adata, 1, bdata, 1);

- return ans;

- }

-

-

-

-

-#endif //HAVE_ATLAS

-

-

-

-} //namespace STK

-#ifndef TNet_Matrix_h

-#define TNet_Matrix_h

-

-#include <stddef.h>

-#include <stdlib.h>

-#include <stdexcept>

-#include <iostream>

-

-#ifdef HAVE_ATLAS

-extern "C"{

- #include <cblas.h>

- #include <clapack.h>

-}

-#endif

-

-#include "Common.h"

-#include "MathAux.h"

-#include "Types.h"

-#include "Error.h"

-

-//#define TRACE_MATRIX_OPERATIONS

-#define CHECKSIZE

-

-namespace TNet

-{

-

-

- // class matrix_error : public std::logic_error {};

- // class matrix_sizes_error : public matrix_error {};

-

- // declare the class so the header knows about it

- template<typename _ElemT> class Vector;

- template<typename _ElemT> class SubVector;

- template<typename _ElemT> class Matrix;

- template<typename _ElemT> class SubMatrix;

-

- // we need to declare the friend << operator here

- template<typename _ElemT>

- std::ostream & operator << (std::ostream & rOut, const Matrix<_ElemT> & rM);

-

- // we need to declare the friend << operator here

- template<typename _ElemT>

- std::istream & operator >> (std::istream & rIn, Matrix<_ElemT> & rM);

-

- // we need to declare this friend function here

- template<typename _ElemT>

- _ElemT TraceOfProduct(const Matrix<_ElemT> &A, const Matrix<_ElemT> &B); // tr(A B)

-

- // we need to declare this friend function here

- template<typename _ElemT>

- _ElemT TraceOfProductT(const Matrix<_ElemT> &A, const Matrix<_ElemT> &B); // tr(A B^T)==tr(A^T B)

-

-

- /** **************************************************************************

- ** **************************************************************************

- * @brief Provides a matrix class

- *

- * This class provides a way to work with matrices in TNet.

- * It encapsulates basic operations and memory optimizations.

- *

- */

- template<typename _ElemT>

- class Matrix

- {

- public:

- /// defines a transpose type

-

- struct HtkHeader

- {

- INT_32 mNSamples;

- INT_32 mSamplePeriod;

- INT_16 mSampleSize;

- UINT_16 mSampleKind;

- };

-

-

- /**

- * @brief Extension of the HTK header

- */

- struct HtkHeaderExt

- {

- INT_32 mHeaderSize;

- INT_32 mVersion;

- INT_32 mSampSize;

- };

-

-

-

-

- /// defines a type of this

- typedef Matrix<_ElemT> ThisType;

-

- // Constructors

-

- /// Empty constructor

- Matrix<_ElemT> ():

- mpData(NULL), mMCols(0), mMRows(0), mStride(0)

-#ifdef STK_MEMALIGN_MANUAL

- , mpFreeData(NULL)

-#endif

- {}

-

- /// Copy constructor

- Matrix<_ElemT> (const Matrix<_ElemT> & rM, MatrixTrasposeType trans=NO_TRANS):

- mpData(NULL)

- { if(trans==NO_TRANS){ Init(rM.mMRows, rM.mMCols); Copy(rM); } else { Init(rM.mMCols,rM.mMRows); Copy(rM,TRANS); } }

-

- /// Copy constructor from another type.

- template<typename _ElemU>

- explicit Matrix<_ElemT> (const Matrix<_ElemU> & rM, MatrixTrasposeType trans=NO_TRANS):

- mpData(NULL)

- { if(trans==NO_TRANS){ Init(rM.Rows(), rM.Cols()); Copy(rM); } else { Init(rM.Cols(),rM.Rows()); Copy(rM,TRANS); } }

-

- /// Basic constructor

- Matrix(const size_t r, const size_t c, bool clear=true)

- { mpData=NULL; Init(r, c, clear); }

-

-

- Matrix<_ElemT> &operator = (const Matrix <_ElemT> &other) { Init(other.Rows(), other.Cols()); Copy(other); return *this; } // Needed for inclusion in std::vector

-

- /// Destructor

- ~Matrix()

- { Destroy(); }

-

-

- /// Initializes matrix (if not done by constructor)

- ThisType &

- Init(const size_t r,

- const size_t c, bool clear=true);

-

- /**

- * @brief Dealocates the matrix from memory and resets the dimensions to (0, 0)

- */

- void

- Destroy();

-

-

- ThisType &

- Zero();

-

- ThisType &

- Unit(); // set to unit.

-

- /**

- * @brief Copies the contents of a matrix

- * @param rM Source data matrix

- * @return Returns reference to this

- */

- template<typename _ElemU> ThisType &

- Copy(const Matrix<_ElemU> & rM, MatrixTrasposeType Trans=NO_TRANS);

-

-

-

- /**

- * @brief Copies the elements of a vector row-by-row into a matrix

- * @param rV Source vector

- * @param nRows Number of rows of returned matrix

- * @param nCols Number of columns of returned matrix

- *

- * Note that rV.Dim() must equal nRows*nCols

- */

- ThisType &

- CopyVectorSplicedRows(const Vector<_ElemT> &rV, const size_t nRows, const size_t nCols);

-

- /**

- * @brief Returns @c true if matrix is initialized

- */

- bool

- IsInitialized() const

- { return mpData != NULL; }

-

- /// Returns number of rows in the matrix

- inline size_t

- Rows() const

- {

- return mMRows;

- }

-

- /// Returns number of columns in the matrix

- inline size_t

- Cols() const

- {

- return mMCols;

- }

-

- /// Returns number of columns in the matrix memory

- inline size_t

- Stride() const

- {

- return mStride;

- }

-

-

- /**

- * @brief Gives access to a specified matrix row without range check

- * @return Pointer to the const array

- */

- inline const _ElemT* __attribute__((aligned(16)))

- pData () const

- {

- return mpData;

- }

-

-

- /**

- * @brief Gives access to a specified matrix row without range check

- * @return Pointer to the non-const data array

- */

- inline _ElemT* __attribute__((aligned(16)))

- pData ()

- {

- return mpData;

- }

-

-

- /**

- * @brief pData_workaround is a workaround that allows SubMatrix to get a

- * @return pointer to non-const data even though the Matrix is const...

- */

- protected:

- inline _ElemT* __attribute__((aligned(16)))

- pData_workaround () const

- {

- return mpData;

- }

- public:

-

-

- /// Returns size of matrix in memory

- size_t

- MSize() const

- {

- return mMRows * mStride * sizeof(_ElemT);

- }

-

- /// Checks the content of the matrix for nan and inf values

- void

- CheckData(const std::string file = "") const

- {

- for(size_t row=0; row<Rows(); row++) {

- for(size_t col=0; col<Cols(); col++) {

- if(isnan((*this)(row,col)) || isinf((*this)(row,col))) {

- std::ostringstream os;

- os << "Invalid value: " << (*this)(row,col)

- << " in matrix row: " << row

- << " col: " << col

- << " file: " << file;

- Error(os.str());

- }

- }

- }

- }

-

- /**

- * **********************************************************************

- * **********************************************************************

- * @defgroup RESHAPE Matrix reshaping rutines

- * **********************************************************************

- * **********************************************************************

- * @{

- */

-

- /**

- * @brief Removes one row from the matrix. The memory is not reallocated.

- */

- ThisType &

- RemoveRow(size_t i);

-

- /** @} */

-

- /**

- * **********************************************************************

- * **********************************************************************

- * @defgroup ACCESS Access functions and operators

- * **********************************************************************

- * **********************************************************************

- * @{

- */

-

- /**

- * @brief Gives access to a specified matrix row without range check

- * @return Subvector object representing the row

- */

- inline const SubVector<_ElemT>

- operator [] (size_t i) const

- {

- assert(i < mMRows);

- return SubVector<_ElemT>(mpData + (i * mStride), Cols());

- }

-

- inline SubVector<_ElemT>

- operator [] (size_t i)

- {

- assert(i < mMRows);

- return SubVector<_ElemT>(mpData + (i * mStride), Cols());

- }

-

- /**

- * @brief Gives access to a specified matrix row without range check

- * @return pointer to the first field of the row

- */

- inline _ElemT*

- pRowData(size_t i)

- {

- assert(i < mMRows);

- return mpData + i * mStride;

- }

-

- /**

- * @brief Gives access to a specified matrix row without range check

- * @return pointer to the first field of the row (const version)

- */

- inline const _ElemT*

- pRowData(size_t i) const

- {

- assert(i < mMRows);

- return mpData + i * mStride;

- }

-

- /**

- * @brief Gives access to matrix elements (row, col)

- * @return reference to the desired field

- */

- inline _ElemT&

- operator () (size_t r, size_t c)

- {

-#ifdef PARANOID

- assert(r < mMRows && c < mMCols);

-#endif

- return *(mpData + r * mStride + c);

- }

-

- /**

- * @brief Gives access to matrix elements (row, col)

- * @return pointer to the desired field (const version)

- */

- inline const _ElemT

- operator () (size_t r, size_t c) const

- {

-#ifdef PARANOID

- assert(r < mMRows && c < mMCols);

-#endif

- return *(mpData + r * mStride + c);

- }

-

- /**

- * @brief Returns a matrix sub-range

- * @param ro Row offset

- * @param r Rows in range

- * @param co Column offset

- * @param c Coluns in range

- * See @c SubMatrix class for details

- */

- SubMatrix<_ElemT>

- Range(const size_t ro, const size_t r,

- const size_t co, const size_t c)

- { return SubMatrix<_ElemT>(*this, ro, r, co, c); }

-

- const SubMatrix<_ElemT>

- Range(const size_t ro, const size_t r,

- const size_t co, const size_t c) const

- { return SubMatrix<_ElemT>(*this, ro, r, co, c); }

- /** @} */

-

-

- /**

- * **********************************************************************

- * **********************************************************************

- * @defgroup MATH ROUTINES

- * **********************************************************************

- * **********************************************************************

- * @{

- **/

-

- /**

- * @brief Returns sum of all elements

- */

- _ElemT&

- Sum() const;

-

- ThisType &

- DotMul(const ThisType& a);

-

- ThisType &

- Scale(_ElemT alpha);

-

- ThisType &

- ScaleCols(const Vector<_ElemT> &scale); // Equivalent to (*this) = (*this) * diag(scale).

-

- ThisType &

- ScaleRows(const Vector<_ElemT> &scale); // Equivalent to (*this) = diag(scale) * (*this);

-

- /// Sum another matrix rMatrix with this matrix

- ThisType&

- Add(const Matrix<_ElemT>& rMatrix);

-

-

- /// Sum scaled matrix rMatrix with this matrix

- ThisType&

- AddScaled(_ElemT alpha, const Matrix<_ElemT>& rMatrix);

-

- /// Apply log to all items of the matrix

- ThisType&

- ApplyLog();

-

- /**

- * @brief Computes the determinant of this matrix

- * @return Returns the determinant of a matrix

- * @ingroup MATH

- *

- */

- _ElemT LogAbsDeterminant(_ElemT *DetSign=NULL);

-

-

- /**

- * @brief Performs matrix inplace inversion

- */

- ThisType &

- Invert(_ElemT *LogDet=NULL, _ElemT *DetSign=NULL, bool inverse_needed=true);

-

- /**

- * @brief Performs matrix inplace inversion in double precision, even if this object is not double precision.

- */

- ThisType &

- InvertDouble(_ElemT *LogDet=NULL, _ElemT *DetSign=NULL, bool inverse_needed=true){

- double LogDet_tmp, DetSign_tmp;

- Matrix<double> dmat(*this); dmat.Invert(&LogDet_tmp, &DetSign_tmp, inverse_needed); if(inverse_needed) (*this).Copy(dmat);

- if(LogDet) *LogDet = LogDet_tmp; if(DetSign) *DetSign = DetSign_tmp;

- return *this;

- }

-

-

- /**

- * @brief Inplace matrix transposition. Applicable only to square matrices

- */

- ThisType &

- Transpose()

- {

- assert(Rows()==Cols());

- size_t M=Rows();

- for(size_t i=0;i<M;i++)

- for(size_t j=0;j<i;j++){

- _ElemT &a = (*this)(i,j), &b = (*this)(j,i);

- std::swap(a,b);

- }

- return *this;

- }

-

-

-

-

-

- bool IsSymmetric(_ElemT cutoff = 1.0e-05) const;

-

- bool IsDiagonal(_ElemT cutoff = 1.0e-05) const;

-

- bool IsUnit(_ElemT cutoff = 1.0e-05) const;

-

- bool IsZero(_ElemT cutoff = 1.0e-05) const;

-

- _ElemT FrobeniusNorm() const; // sqrt of sum of square elements.

-

- _ElemT LargestAbsElem() const; // largest absolute value.

-

-

- friend _ElemT TNet::TraceOfProduct<_ElemT>(const Matrix<_ElemT> &A, const Matrix<_ElemT> &B); // tr(A B)

- friend _ElemT TNet::TraceOfProductT<_ElemT>(const Matrix<_ElemT> &A, const Matrix<_ElemT> &B); // tr(A B^T)==tr(A^T B)

- friend class SubMatrix<_ElemT>; // so it can get around const restrictions on the pointer to mpData.

-

- /** **********************************************************************

- * **********************************************************************

- * @defgroup BLAS_ROUTINES BLAS ROUTINES

- * @ingroup MATH

- * **********************************************************************

- * **********************************************************************

- **/

-

- ThisType &

- BlasGer(const _ElemT alpha, const Vector<_ElemT>& rA, const Vector<_ElemT>& rB);

-

- ThisType &

- Axpy(const _ElemT alpha, const Matrix<_ElemT> &rM, MatrixTrasposeType transA=NO_TRANS);

-

- ThisType &

- BlasGemm(const _ElemT alpha,

- const ThisType& rA, MatrixTrasposeType transA,

- const ThisType& rB, MatrixTrasposeType transB,

- const _ElemT beta = 0.0);

-

-

- /** @} */

-

-

- /** **********************************************************************

- * **********************************************************************

- * @defgroup IO Input/Output ROUTINES

- * **********************************************************************

- * **********************************************************************

- * @{

- **/

-

- friend std::ostream &

- operator << <> (std::ostream & out, const ThisType & m);

-

- void PrintOut(char *file);

- void ReadIn(char *file);

-

-

- bool

- LoadHTK(const char* pFileName);

-

- /** @} */

-

-

- protected:

-// inline void swap4b(void *a);

-// inline void swap2b(void *a);

-

-

- protected:

- /// data memory area

- _ElemT* mpData;

-

- /// these atributes store the real matrix size as it is stored in memory

- /// including memalignment

- size_t mMCols; ///< Number of columns

- size_t mMRows; ///< Number of rows

- size_t mStride; ///< true number of columns for the internal matrix.

- ///< This number may differ from M_cols as memory

- ///< alignment might be used

-

-#ifdef STK_MEMALIGN_MANUAL

- /// data to be freed (in case of manual memalignment use, see Common.h)

- _ElemT* mpFreeData;

-#endif

- }; // class Matrix

-

- template<> Matrix<float> & Matrix<float>::Invert(float *LogDet, float *DetSign, bool inverse_needed); // state that we will implement separately for float and double.

- template<> Matrix<double> & Matrix<double>::Invert(double *LogDet, double *DetSign, bool inverse_needed);

-

-

-

- /** **************************************************************************

- ** **************************************************************************

- * @brief Sub-matrix representation

- *

- * This class provides a way to work with matrix cutouts in STK.

- *

- *

- */

- template<typename _ElemT>

- class SubMatrix : public Matrix<_ElemT>

- {

- typedef SubMatrix<_ElemT> ThisType;

-

- public:

- /// Constructor

- SubMatrix(const Matrix<_ElemT>& rT, // Input matrix cannot be const because SubMatrix can change its contents.

- const size_t ro,

- const size_t r,

- const size_t co,

- const size_t c);

-

-

- /// The destructor

- ~SubMatrix<_ElemT>()

- {

-#ifndef STK_MEMALIGN_MANUAL

- Matrix<_ElemT>::mpData = NULL;

-#else

- Matrix<_ElemT>::mpFreeData = NULL;

-#endif

- }

-

- /// Assign operator

- ThisType& operator=(const ThisType& rSrc)

- {

- //std::cout << "[PERFORMing operator= SubMatrix&^2]" << std::flush;

- this->mpData = rSrc.mpData;

- this->mMCols = rSrc.mMCols;

- this->mMRows = rSrc.mMRows;

- this->mStride = rSrc.mStride;

- this->mpFreeData = rSrc.mpFreeData;

- return *this;

- }

-

-

-

- /// Initializes matrix (if not done by constructor)

- ThisType &

- Init(const size_t r,

- const size_t c, bool clear=true)

- { Error("Submatrix cannot do Init"); return *this; }

-

- /**

- * @brief Dealocates the matrix from memory and resets the dimensions to (0, 0)

- */

- void

- Destroy()

- { Error("Submatrix cannot do Destroy"); }

-

-

-

- };

-

-

-

- //Create useful shortcuts

- typedef Matrix<BaseFloat> BfMatrix;

- typedef SubMatrix<BaseFloat> BfSubMatrix;

-

- /**

- * Function for summing matrices of different types

- */

- template<typename _ElemT, typename _ElemU>

- void Add(Matrix<_ElemT>& rDst, const Matrix<_ElemU>& rSrc) {

- assert(rDst.Cols() == rSrc.Cols());

- assert(rDst.Rows() == rSrc.Rows());

-

- for(size_t i=0; i<rDst.Rows(); i++) {

- const _ElemU* p_src = rSrc.pRowData(i);

- _ElemT* p_dst = rDst.pRowData(i);

- for(size_t j=0; j<rDst.Cols(); j++) {

- *p_dst++ += (_ElemT)*p_src++;

- }

- }

- }

-

- /**

- * Function for summing matrices of different types

- */

- template<typename _ElemT, typename _ElemU>

- void AddScaled(Matrix<_ElemT>& rDst, const Matrix<_ElemU>& rSrc, _ElemT scale) {

- assert(rDst.Cols() == rSrc.Cols());

- assert(rDst.Rows() == rSrc.Rows());

-

- Vector<_ElemT> tmp(rDst[0]);

-

- for(size_t i=0; i<rDst.Rows(); i++) {

- tmp.Copy(rSrc[i]);

- rDst[i].BlasAxpy(scale, tmp);

-

- /*

- const _ElemU* p_src = rSrc.pRowData(i);

- _ElemT* p_dst = rDst.pRowData(i);

- for(size_t j=0; j<rDst.Cols(); j++) {

- *p_dst++ += (_ElemT)(*p_src++) * scale;

- }

- */

- }

- }

-

-

-

-

-

-} // namespace STK

-

-

-

-//*****************************************************************************

-//*****************************************************************************

-// we need to include the implementation

-#include "Matrix.tcc"

-//*****************************************************************************

-//*****************************************************************************

-

-

-/******************************************************************************

- ******************************************************************************

- * The following section contains specialized template definitions

- * whose implementation is in Matrix.cc

- */

-

-

-//#ifndef TNet_Matrix_h

-#endif

-

-/** @file Matrix.tcc

- * This is an internal header file, included by other library headers.

- * You should not attempt to use it directly.

- */

-

-

-#ifndef TNet_Matrix_tcc

-#define TNet_Matrix_tcc

-

-//#pragma GCC system_header

-

-#include <cstdlib>

-#include <cmath>

-#include <cfloat>

-#include <fstream>

-#include <iomanip>

-#include <typeinfo>

-#include <algorithm>

-#include <limits>

-#include <vector>

-#include "Common.h"

-

-#ifndef _XOPEN_SOURCE

- #define _XOPEN_SOURCE 600

-#endif

-

-

-#ifdef HAVE_ATLAS

-extern "C"{

- #include <cblas.h>

-}

-#endif

-

-

-#include "Common.h"

-#include "Vector.h"

-namespace TNet

-{

-

-//******************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT> &

- Matrix<_ElemT>::

- Init(const size_t rows,

- const size_t cols,

- bool clear)

- {

- if(mpData != NULL) Destroy();

- if(rows*cols == 0){

- assert(rows==0 && cols==0);

- mMRows=rows;

- mMCols=cols;

-#ifdef STK_MEMALIGN_MANUAL

- mpFreeData=NULL;

-#endif

- mpData=NULL;

- return *this;

- }

- // initialize some helping vars

- size_t skip;

- size_t real_cols;

- size_t size;

- void* data; // aligned memory block

- void* free_data; // memory block to be really freed

-

- // compute the size of skip and real cols

- skip = ((16 / sizeof(_ElemT)) - cols % (16 / sizeof(_ElemT))) % (16 / sizeof(_ElemT));

- real_cols = cols + skip;

- size = rows * real_cols * sizeof(_ElemT);

-

- // allocate the memory and set the right dimensions and parameters

-

- if (NULL != (data = stk_memalign(16, size, &free_data)))

- {

- mpData = static_cast<_ElemT *> (data);

-#ifdef STK_MEMALIGN_MANUAL

- mpFreeData = static_cast<_ElemT *> (free_data);

-#endif

- mMRows = rows;

- mMCols = cols;

- mStride = real_cols;

- }

- else

- {

- throw std::bad_alloc();

- }

- if(clear) Zero();

- return *this;

- } //

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- template<typename _ElemU>

- Matrix<_ElemT> &

- Matrix<_ElemT>::

- Copy(const Matrix<_ElemU> & rM, MatrixTrasposeType Trans)

- {

- if(Trans==NO_TRANS){

- assert(mMRows == rM.Rows() && mMCols == rM.Cols());

- for(size_t i = 0; i < mMRows; i++)

- (*this)[i].Copy(rM[i]);

- return *this;

- } else {

- assert(mMCols == rM.Rows() && mMRows == rM.Cols());

- for(size_t i = 0; i < mMRows; i++)

- for(size_t j = 0; j < mMCols; j++)

- (*this)(i,j) = rM(j,i);

- return *this;

- }

- }

-

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT> &

- Matrix<_ElemT>::

- CopyVectorSplicedRows(const Vector<_ElemT> &rV, const size_t nRows, const size_t nCols) {

- assert(rV.Dim() == nRows*nCols);

- mMRows = nRows;

- mMCols = nCols;

- Init(nRows,nCols,true);

- for(size_t r=0; r<mMRows; r++)

- for(size_t c=0; c<mMCols; c++)

- (*this)(r,c) = rV(r*mMCols + c);

-

- return *this;

- }

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT> &

- Matrix<_ElemT>::

- RemoveRow(size_t i)

- {

- assert(i < mMRows && "Access out of matrix");

- for(size_t j = i + 1; j < mMRows; j++)

- (*this)[j - 1].Copy((*this)[j]);

- mMRows--;

- return *this;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- // The destructor

- template<typename _ElemT>

- void

- Matrix<_ElemT>::

- Destroy()

- {

- // we need to free the data block if it was defined

-#ifndef STK_MEMALIGN_MANUAL

- if (NULL != mpData) free(mpData);

-#else

- if (NULL != mpData) free(mpFreeData);

- mpFreeData = NULL;

-#endif

-

- mpData = NULL;

- mMRows = mMCols = 0;

- }

-

- //****************************************************************************

- //****************************************************************************

-// template<typename _ElemT>

-// void

-// Matrix<_ElemT>::

-// VectorizeRows(Vector<_ElemT> &rV) {

-//#ifdef PARANIOD

-// assert(rV.Dim() == mMRows*mMCols);

-//#endif

-// for(size_t r=0; r<mMRows; r++) {

-// rV.Range((r-1)*mMCols, mMCols).Copy((*this)[r]);

-// }

-// }

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- bool

- Matrix<_ElemT>::

- LoadHTK(const char* pFileName)

- {

- HtkHeader htk_hdr;

-

- FILE *fp = fopen(pFileName, "rb");

- if(!fp)

- {

- return false;

- }

-

- read(fileno(fp), &htk_hdr, sizeof(htk_hdr));

-

- swap4(htk_hdr.mNSamples);

- swap4(htk_hdr.mSamplePeriod);

- swap2(htk_hdr.mSampleSize);

- swap2(htk_hdr.mSampleKind);

-

- Init(htk_hdr.mNSamples, htk_hdr.mSampleSize / sizeof(float));

-

- size_t i;

- size_t j;

- if (typeid(_ElemT) == typeid(float))

- {

- for (i=0; i< Rows(); ++i) {

- read(fileno(fp), (*this).pRowData(i), Cols() * sizeof(float));

-

- for(j = 0; j < Cols(); j++) {

- swap4(((*this)(i,j)));

- }

- }

- }

- else

- {

- float *pmem = new (std::nothrow) float[Cols()];

- if (!pmem)

- {

- fclose(fp);

- return false;

- }

-

- for(i = 0; i < Rows(); i++) {

- read(fileno(fp), pmem, Cols() * sizeof(float));

-

- for (j = 0; j < Cols(); ++j) {

- swap4(pmem[j]);

- (*this)(i,j) = static_cast<_ElemT>(pmem[j]);

- }

- }

- delete [] pmem;

- }

-

- fclose(fp);

-

- return true;

- }

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT> &

- Matrix<_ElemT>::

- DotMul(const ThisType& a)

- {

- size_t i;

- size_t j;

-

- for (i = 0; i < mMRows; ++i) {

- for (j = 0; j < mMCols; ++j) {

- (*this)(i,j) *= a(i,j);

- }

- }

- return *this;

- }

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- _ElemT &

- Matrix<_ElemT>::

- Sum() const

- {

- double sum = 0.0;

-

- for (size_t i = 0; i < Rows(); ++i) {

- for (size_t j = 0; j < Cols(); ++j) {

- sum += (*this)(i,j);

- }

- }

-

- return sum;

- }

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT>&

- Matrix<_ElemT>::

- Scale(_ElemT alpha)

- {

-#if 0

- for (size_t i = 0; i < Rows(); ++i)

- for (size_t j = 0; j < Cols(); ++j)

- (*this)(i,j) *= alpha;

-#else

- for (size_t i = 0; i < Rows(); ++i) {

- _ElemT* p_data = pRowData(i);

- for (size_t j = 0; j < Cols(); ++j) {

- *p_data++ *= alpha;

- }

- }

-#endif

- return *this;

- }

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT>&

- Matrix<_ElemT>::

- ScaleRows(const Vector<_ElemT>& scale) // scales each row by scale[i].

- {

- assert(scale.Dim() == Rows());

- size_t M = Rows(), N = Cols();

-

- for (size_t i = 0; i < M; i++) {

- _ElemT this_scale = scale(i);

- for (size_t j = 0; j < N; j++) {

- (*this)(i,j) *= this_scale;

- }

- }

- return *this;

- }

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT>&

- Matrix<_ElemT>::

- ScaleCols(const Vector<_ElemT>& scale) // scales each column by scale[i].

- {

- assert(scale.Dim() == Cols());

- for (size_t i = 0; i < Rows(); i++) {

- for (size_t j = 0; j < Cols(); j++) {

- _ElemT this_scale = scale(j);

- (*this)(i,j) *= this_scale;

- }

- }

- return *this;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT>&

- Matrix<_ElemT>::

- Add(const Matrix<_ElemT>& rMatrix)

- {

- assert(rMatrix.Cols() == Cols());

- assert(rMatrix.Rows() == Rows());

-

-#if 0

- //this can be slow

- for (size_t i = 0; i < Rows(); i++) {

- for (size_t j = 0; j < Cols(); j++) {

- (*this)(i,j) += rMatrix(i,j);

- }

- }

-#else

- //this will be faster (but less secure)

- for(size_t i=0; i<Rows(); i++) {

- const _ElemT* p_src = rMatrix.pRowData(i);

- _ElemT* p_dst = pRowData(i);

- for(size_t j=0; j<Cols(); j++) {

- *p_dst++ += *p_src++;

- }

- }

-#endif

- return *this;

- }

-

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT>&

- Matrix<_ElemT>::

- AddScaled(_ElemT alpha, const Matrix<_ElemT>& rMatrix)

- {

- assert(rMatrix.Cols() == Cols());

- assert(rMatrix.Rows() == Rows());

-

-#if 0

- //this can be slow

- for (size_t i = 0; i < Rows(); i++) {

- for (size_t j = 0; j < Cols(); j++) {

- (*this)(i,j) += rMatrix(i,j) * alpha;

- }

- }

-#else

- /*

- //this will be faster (but less secure)

- for(size_t i=0; i<Rows(); i++) {

- const _ElemT* p_src = rMatrix.pRowData(i);

- _ElemT* p_dst = pRowData(i);

- for(size_t j=0; j<Cols(); j++) {

- *p_dst++ += *p_src++ * alpha;

- }

- }

- */

-

- //let's use BLAS

- for(size_t i=0; i<Rows(); i++) {

- (*this)[i].BlasAxpy(alpha, rMatrix[i]);

- }

-#endif

- return *this;

- }

-

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT>&

- Matrix<_ElemT>::

- ApplyLog()

- {

-

-#if 0

- //this can be slow

- for (size_t i = 0; i < Rows(); i++) {

- for (size_t j = 0; j < Cols(); j++) {

- (*this)(i,j) = += _LOG((*this)(i,j));

- }

- }

-#else

- //this will be faster (but less secure)

- for(size_t i=0; i<Rows(); i++) {

- _ElemT* p_data = pRowData(i);

- for(size_t j=0; j<Cols(); j++) {

- *p_data = _LOG(*p_data);

- p_data++;

- }

- }

-#endif

- return *this;

- }

-

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT> &

- Matrix<_ElemT>::

- Zero()

- {

- for(size_t row=0;row<mMRows;row++)

- memset(mpData + row*mStride, 0, sizeof(_ElemT)*mMCols);

- return *this;

- }

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- Matrix<_ElemT> &

- Matrix<_ElemT>::

- Unit()

- {

- for(size_t row=0;row<std::min(mMRows,mMCols);row++){

- memset(mpData + row*mStride, 0, sizeof(_ElemT)*mMCols);

- (*this)(row,row) = 1.0;

- }

- return *this;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- void

- Matrix<_ElemT>::

- PrintOut(char* file)

- {

- FILE* f = fopen(file, "w");

- unsigned i,j;

- fprintf(f, "%dx%d\n", this->mMRows, this->mMCols);

-

- for(i=0; i<this->mMRows; i++)

- {

- _ElemT* row = (*this)[i];

-

- for(j=0; j<this->mStride; j++){

- fprintf(f, "%20.17f ",row[j]);

- }

- fprintf(f, "\n");

- }

-

- fclose(f);

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- void

- Matrix<_ElemT>::

- ReadIn(char* file)

- {

- FILE* f = fopen(file, "r");

- int i = 0;

- int j = 0;

- fscanf(f, "%dx%d\n", &i,&j);

- fprintf(stderr, "%dx%d\n", i,j);

-

- for(i=0; i<this->mMRows; i++)

- {

- _ElemT* row = (*this)[i];

-

- for(j=0; j<this->mStride; j++){

- fscanf(f, "%f ",&row[j]);

- }

- //fprintf(f, "\n");

- }

-

- fclose(f);

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- void Save (std::ostream &rOut, const Matrix<_ElemT> &rM)

- {

- for (size_t i = 0; i < rM.Rows(); i++) {

- for (size_t j = 0; j < rM.Cols(); j++) {

- rOut << rM(i,j) << ' ';

- }

- rOut << '\n';

- }

- if(rOut.fail())

- throw std::runtime_error("Failed to write matrix to stream");

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- std::ostream &

- operator << (std::ostream & rOut, const Matrix<_ElemT> & rM)

- {

- rOut << "m " << rM.Rows() << ' ' << rM.Cols() << '\n';

- Save(rOut, rM);

- return rOut;

- }

-

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- void Load (std::istream & rIn, Matrix<_ElemT> & rM)

- {

- if(MatrixVectorIostreamControl::Flags(rIn, ACCUMULATE_INPUT)) {

- for (size_t i = 0; i < rM.Rows(); i++) {

- std::streamoff pos = rIn.tellg();

- for (size_t j = 0; j < rM.Cols(); j++) {

- _ElemT tmp;

- rIn >> tmp;

- rM(i,j) += tmp;

- if(rIn.fail()){

- throw std::runtime_error("Failed to read matrix from stream. File position is "+to_string(pos));

- }

- }

- }

- } else {

- for (size_t i = 0; i < rM.Rows(); i++) {

- std::streamoff pos = rIn.tellg();

- for (size_t j = 0; j < rM.Cols(); j++) {

- rIn >> rM(i,j);

- if(rIn.fail()){

- throw std::runtime_error("Failed to read matrix from stream. File position is "+to_string(pos));

- }

-

- }

- }

- }

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<typename _ElemT>

- std::istream &

- operator >> (std::istream & rIn, Matrix<_ElemT> & rM)

- {

- while(isascii(rIn.peek()) && isspace(rIn.peek())) rIn.get(); // eat up space.

- if(rIn.peek() == 'm'){ // "new" format: m <nrows> <ncols> \n 1.0 0.2 4.3 ...

- rIn.get();// eat up the 'm'.

- long long int nrows=-1; rIn>>nrows;

- long long int ncols=-1; rIn>>ncols;

- if(rIn.fail()||nrows<0||ncols<0){ throw std::runtime_error("Failed to read matrix from stream: no size\n"); }

-

- size_t nrows2 = size_t(nrows), ncols2 = size_t(ncols);

- assert((long long int)nrows2 == nrows && (long long int)ncols2 == ncols);

-

- if(rM.Rows()!=nrows2 || rM.Cols()!=ncols2) rM.Init(nrows2,ncols2);

- }

- Load(rIn,rM);

- return rIn;

- }

-

-

-

- //****************************************************************************

- //****************************************************************************

- // Constructor

- template<typename _ElemT>

- SubMatrix<_ElemT>::

- SubMatrix(const Matrix<_ElemT>& rT, // Matrix cannot be const because SubMatrix can change its contents. Would have to have a ConstSubMatrix or something...

- const size_t ro,

- const size_t r,

- const size_t co,

- const size_t c)

- {

- assert(ro >= 0 && ro <= rT.Rows());

- assert(co >= 0 && co <= rT.Cols());

- assert(r > 0 && r <= rT.Rows() - ro);

- assert(c > 0 && c <= rT.Cols() - co);

- // point to the begining of window

- Matrix<_ElemT>::mMRows = r;

- Matrix<_ElemT>::mMCols = c;

- Matrix<_ElemT>::mStride = rT.Stride();

- Matrix<_ElemT>::mpData = rT.pData_workaround() + co + ro * rT.Stride();

- }

-

-

-

-#ifdef HAVE_ATLAS

-

- template<>

- Matrix<float> &

- Matrix<float>::

- BlasGer(const float alpha, const Vector<float>& rA, const Vector<float>& rB);

-

-

- template<>

- Matrix<double> &

- Matrix<double>::

- BlasGer(const double alpha, const Vector<double>& rA, const Vector<double>& rB);

-

-

- template<>

- Matrix<float>&

- Matrix<float>::

- BlasGemm(const float alpha,

- const Matrix<float>& rA, MatrixTrasposeType transA,

- const Matrix<float>& rB, MatrixTrasposeType transB,

- const float beta);

-

- template<>

- Matrix<double>&

- Matrix<double>::

- BlasGemm(const double alpha,

- const Matrix<double>& rA, MatrixTrasposeType transA,

- const Matrix<double>& rB, MatrixTrasposeType transB,

- const double beta);

-

- template<>

- Matrix<float>&

- Matrix<float>::

- Axpy(const float alpha,

- const Matrix<float>& rA, MatrixTrasposeType transA);

-

- template<>

- Matrix<double>&

- Matrix<double>::

- Axpy(const double alpha,

- const Matrix<double>& rA, MatrixTrasposeType transA);

-

- template <> // non-member so automatic namespace lookup can occur.

- double TraceOfProduct(const Matrix<double> &A, const Matrix<double> &B);

-

- template <> // non-member so automatic namespace lookup can occur.

- double TraceOfProductT(const Matrix<double> &A, const Matrix<double> &B);

-

- template <> // non-member so automatic namespace lookup can occur.

- float TraceOfProduct(const Matrix<float> &A, const Matrix<float> &B);

-

- template <> // non-member so automatic namespace lookup can occur.

- float TraceOfProductT(const Matrix<float> &A, const Matrix<float> &B);

-

-

-

-#else // HAVE_ATLAS

- #error Routines in this section are not implemented yet without BLAS

-#endif // HAVE_ATLAS

-

- template<class _ElemT>

- bool

- Matrix<_ElemT>::

- IsSymmetric(_ElemT cutoff) const {

- size_t R=Rows(), C=Cols();

- if(R!=C) return false;

- _ElemT bad_sum=0.0, good_sum=0.0;

- for(size_t i=0;i<R;i++){

- for(size_t j=0;j<i;j++){

- _ElemT a=(*this)(i,j),b=(*this)(j,i), avg=0.5*(a+b), diff=0.5*(a-b);

- good_sum += fabs(avg); bad_sum += fabs(diff);

- }

- good_sum += fabs((*this)(i,i));

- }

- if(bad_sum > cutoff*good_sum) return false;

- return true;

- }

-

- template<class _ElemT>

- bool

- Matrix<_ElemT>::

- IsDiagonal(_ElemT cutoff) const{

- size_t R=Rows(), C=Cols();

- _ElemT bad_sum=0.0, good_sum=0.0;

- for(size_t i=0;i<R;i++){

- for(size_t j=0;j<C;j++){

- if(i==j) good_sum += (*this)(i,j);

- else bad_sum += (*this)(i,j);

- }

- }

- return (!(bad_sum > good_sum * cutoff));

- }

-

- template<class _ElemT>

- bool

- Matrix<_ElemT>::

- IsUnit(_ElemT cutoff) const {

- size_t R=Rows(), C=Cols();

- if(R!=C) return false;

- _ElemT bad_sum=0.0;

- for(size_t i=0;i<R;i++)

- for(size_t j=0;j<C;j++)

- bad_sum += fabs( (*this)(i,j) - (i==j?1.0:0.0));

- return (bad_sum <= cutoff);

- }

-

- template<class _ElemT>

- bool

- Matrix<_ElemT>::

- IsZero(_ElemT cutoff)const {

- size_t R=Rows(), C=Cols();

- _ElemT bad_sum=0.0;

- for(size_t i=0;i<R;i++)

- for(size_t j=0;j<C;j++)

- bad_sum += fabs( (*this)(i,j) );

- return (bad_sum <= cutoff);

- }

-

- template<class _ElemT>

- _ElemT

- Matrix<_ElemT>::

- FrobeniusNorm() const{

- size_t R=Rows(), C=Cols();

- _ElemT sum=0.0;

- for(size_t i=0;i<R;i++)

- for(size_t j=0;j<C;j++){

- _ElemT tmp = (*this)(i,j);

- sum += tmp*tmp;

- }

- return sqrt(sum);

- }

-

- template<class _ElemT>

- _ElemT

- Matrix<_ElemT>::

- LargestAbsElem() const{

- size_t R=Rows(), C=Cols();

- _ElemT largest=0.0;

- for(size_t i=0;i<R;i++)

- for(size_t j=0;j<C;j++)

- largest = std::max(largest, (_ElemT)fabs((*this)(i,j)));

- return largest;

- }

-

-

-

- // Uses SVD to compute the eigenvalue decomposition of a symmetric positive semidefinite

- // matrix:

- // (*this) = rU * diag(rS) * rU^T, with rU an orthogonal matrix so rU^{-1} = rU^T.

- // Does this by computing svd (*this) = U diag(rS) V^T ... answer is just U diag(rS) U^T.

- // Throws exception if this failed to within supplied precision (typically because *this was not

- // symmetric positive definite).

-

-

-

- template<class _ElemT>

- _ElemT

- Matrix<_ElemT>::

- LogAbsDeterminant(_ElemT *DetSign){

- _ElemT LogDet;

- Matrix<_ElemT> tmp(*this);

- tmp.Invert(&LogDet, DetSign, false); // false== output not needed (saves some computation).

- return LogDet;

- }

-

-}// namespace TNet

-

-// #define TNet_Matrix_tcc

-#endif

-#include "MlfStream.h"

-#include "Common.h"

-#include "Error.h"

-

-

-namespace TNet

-{

- //******************************************************************************

- LabelContainer::

- ~LabelContainer()

- {

- while (!this->mLabelList.empty())

- {

- delete this->mLabelList.back();

- this->mLabelList.pop_back();

- }

- }

-

- //******************************************************************************

- size_t

- LabelContainer::

- DirDepth(const std::string & rPath)

- {

- size_t depth = 0;

- size_t length = rPath.length();

- const char * s = rPath.c_str();

-

- for (size_t i = 0; i < length; i++)

- {

- if (*s == '/' || *s == '\\')

- {

- depth++;

- }

- s++;

- }

- return depth;

- }

-

-

- //******************************************************************************

- void

- LabelContainer::

- Insert(const std::string & rLabel,

- std::streampos Pos)

- {

- LabelRecord ls;

- size_t depth;

- LabelRecord tmp_ls;

-

- // we need to compute the depth of the label path if

- // wildcard is used

- // do we have a wildcard???

- if (rLabel[0] == '*')

- {

- depth = this->DirDepth(rLabel);

- }

- else

- {

- depth = MAX_LABEL_DEPTH;

- }

-

- // perhaps we want to store the depth of the path in the label for the wildcards

- // to work

- this->mDepths.insert(depth);

-

- // store the values

- ls.mStreamPos = Pos;

- ls.miLabelListLimit = mLabelList.end();

-

-

- if (mLabelList.begin() != mLabelList.end()) {

- ls.miLabelListLimit--;

- }

-

- // if no wildcard chars, then we try to store in hash, otherwise store in

- // list

- if (rLabel.find_first_of("*?%",1) == rLabel.npos)

- {

- if (!Find(rLabel, tmp_ls))

- {

- // look in the

- this->mLabelMap[rLabel] = ls;

- }

- else {

- ;

- //Warning("More general definition found when inserting " + rLabel + " ... label: " + MatchedPattern());

- }

- }

- else

- {

- this->mLabelList.push_back(new std::pair<std::string,LabelRecord>(rLabel, ls));

- }

- }

-

-

- //******************************************************************************

- bool

- LabelContainer::

- FindInHash(const std::string & rLabel, LabelRecord & rLS)

- {

- bool found = false;

-

- std::string str;

-

- // current depth within the str

- DepthType current_depth = MAX_LABEL_DEPTH;

-

- // current search position within the str

- size_t prev = rLabel.size() + 1;

-

- // we will walk through the set depts bacwards so we begin at the end and move

- // to the front...

- std::set<DepthType>::reverse_iterator ri (this->mDepths.end());

- std::set<DepthType>::reverse_iterator rlast (this->mDepths.begin());

- LabelHashType::iterator lab;

-

- // we perform the search until we run to the end of the set or we find something

- while ((!found) && (ri != rlast))

- {

- // we don't need to do anything with the string if the depth is set to

- // max label depth since it contains no *

- if (*ri == MAX_LABEL_DEPTH)

- {

- found = ((lab=this->mLabelMap.find(rLabel)) != this->mLabelMap.end());

- if (found) str = rLabel;

- }

- // we will crop the string and put * in the begining and try to search

- else

- {

- // we know that we walk backwards in the depths, so we need to first find

- // the last / and

- if (current_depth == MAX_LABEL_DEPTH)

- {

- if (*ri > 0)

- {

- // we find the ri-th / from back

- for (DepthType i=1; (i <= *ri) && (prev != rLabel.npos); i++)

- {

- prev = rLabel.find_last_of("/\\", prev-1);

- }

- }

- else

- {

- prev = 0;

- }

-

- // check if finding succeeded (prev == str.npos => failure, see STL)

- if (prev != rLabel.npos)

- {

- // construct the new string beign sought for

- str.assign(rLabel, prev, rLabel.size());

- str = '*' + str;

-

- // now we try to find

- found = ((lab=this->mLabelMap.find(str)) != this->mLabelMap.end());

-

- // say, that current depth is *ri

- current_depth = *ri;

- }

- else

- {

- prev = rLabel.size() + 1;

- }

- } // if (current_depth == MAX_LABEL_DEPTH)

- else

- {

- // now we know at which / we are from the back, so we search forward now

- // and we need to reach the ri-th /

- while (current_depth > *ri)

- {

- // we try to find next /

- if ((prev = rLabel.find_first_of("/\\", prev+1)) != rLabel.npos)

- current_depth--;

- else

- return false;

- }

-

- // construct the new string beign sought for

- str.assign(rLabel, prev, rLabel.size());

- str = '*' + str;

-

- // now we try to find

- found = ((lab=this->mLabelMap.find(str)) != this->mLabelMap.end());

- }

- }

-

- // move one element further (jump to next observed depth)

- ri++;

- } // while (run)

-

- // some debug info

- if (found)

- {

- rLS = lab->second;

- this->mMatchedPattern = str;

- }

-

- return found;

- }

-

-

- //******************************************************************************

- bool

- LabelContainer::

- FindInList(const std::string & rLabel, LabelRecord & rLS, bool limitSearch)

- {

-

- bool found = false;

- std::string str;

- LabelListType::iterator lab = mLabelList.begin();

- LabelListType::iterator limit;

-

- if (limitSearch && (rLS.miLabelListLimit != mLabelList.end()))

- {

- limit = rLS.miLabelListLimit;

- limit++;

- }

- else

- {

- limit = this->mLabelList.end();

- }

-

- // we perform sequential search until we run to the end of the list or we find

- // something

- while ((!found) && (lab != limit))

- {

- if (ProcessMask(rLabel, (*lab)->first, str))

- {

- found = true;

- }

- else

- {

- lab++;

- }

- } // while (run)

-

- // some debug info

- if (found)

- {

- rLS = (*lab)->second;

- this->mMatchedPattern = (*lab)->first;

- this->mMatchedPatternMask = str;

- }

- return found;

- }

-

-

- //******************************************************************************

- bool

- LabelContainer::

- Find(const std::string & rLabel, LabelRecord & rLS)

- {

- // try to find the label in the Hash

- if (FindInHash(rLabel, rLS))

- {

- // we look in the list, but we limit the search.

- FindInList(rLabel, rLS, true);

- return true;

- } //if (this->mLabelContainer.FindInHash(rLabel, label_stream))

- else

- {

- // we didn't find it in the hash so we look in the list

- return FindInList(rLabel, rLS);

- }

- }

-

-} // namespace TNet

-

-/** @file MlfStream.h

- * This is an TNet C++ Library header.

- *

- * The naming convention in this file coppies the std::* naming as well as STK

- */

-

-

-#ifndef STK_MlfStream_h

-#define STK_MlfStream_h

-

-#include <iostream>

-#include <vector>

-#include <map>

-#include <list>

-#include <set>

-

-

-namespace TNet

-{

- class LabelRecord;

- class LabelContainer;

-

-

- /// this container stores the lables in linear order as they came

- /// i.e. they cannot be hashed

- typedef std::list< std::pair<std::string,LabelRecord> *> LabelListType;

-

- /// type of the container used to store the labels

- typedef std::map<std::string, LabelRecord> LabelHashType;

-

-

-

- /**

- * @brief Describes type of MLF definition

- *

- * See HTK book for MLF structure. Terms used in TNet are

- * compatible with those in HTK book.

- */

- enum MlfDefType

- {

- MLF_DEF_UNKNOWN = 0, ///< unknown definition

- MLF_DEF_IMMEDIATE_TRANSCRIPTION, ///< immediate transcription

- MLF_DEF_SUB_DIR_DEF ///< subdirectory definition

- };

-

-

-

- /** **************************************************************************

- * @brief Holds association between label and stream

- */

- class LabelRecord

- {

-

- public:

- LabelRecord() : miLabelListLimit(NULL)

- { }

-

- ~LabelRecord()

- { }

-

- /// definition type

- MlfDefType mDefType;

-

- /// position of the label in the stream

- std::streampos mStreamPos;

-

- /**

- * @brief points to the current end of the LabelList

- *

- * The reason for storing this value is to know when we inserted

- * a label into the hash. It is possible, that the hash label came

- * after list label, in which case the list label is prefered

- */

- LabelListType::iterator miLabelListLimit;

-

- };

-

-

-

-

- /**

- * @brief Provides an interface to label hierarchy and searching

- *

- * This class stores label files in a map structure. When a wildcard

- * convence is used, the class stores the labels in separate maps according

- * to level of wildcard abstraction. By level we mean the directory structure

- * depth.

- */

- class LabelContainer

- {

- public:

- /// The constructor

- LabelContainer() : mUseHashedSearch(true) {}

-

- /// The destructor

- ~LabelContainer();

-

- /**

- * @brief Inserts new label to the hash structure

- */

- void

- Insert(

- const std::string & rLabel,

- std::streampos Pos);

-

-

- /**

- * @brief Looks for a record in the hash

- */

- bool

- FindInHash(

- const std::string& rLabel,

- LabelRecord& rLS);

-

- /**

- * @brief Looks for a record in the list

- * @param rLabel Label to look for

- * @param rLS Structure to fill with found data

- * @param limitSearch If true @p rLS's @c mLabelListLimit gives the limiting position in the list

- */

- bool

- FindInList(

- const std::string& rLabel,

- LabelRecord& rLS,

- bool limitSearch = false);

-

- /**

- * @brief Looks for a record

- */

- bool

- Find(

- const std::string & rLabel,

- LabelRecord & rLS);

-

- /**

- * @brief Returns the matched pattern

- */

- const std::string &

- MatchedPattern() const

- {

- return mMatchedPattern;

- }

-

- /**

- * @brief Returns the matched pattern mask (%%%)

- */

- const std::string &

- MatchedPatternMask() const

- {

- return mMatchedPatternMask;

- }

-

- /**

- * @brief Writes contents to stream (text)

- * @param rOStream stream to write to

- */

- void

- Write(std::ostream& rOStream);

-

- private:

- /// type used for directory depth notation

- typedef size_t DepthType;

-

-

- /// this set stores depths of * labels observed at insertion

- std::set<DepthType> mDepths;

-

- /// stores the labels

- LabelHashType mLabelMap;

- LabelListType mLabelList;

-

- /// true if labels are to be sought by hashing function (fast) or by

- /// sequential search (slow)

- bool mUseHashedSearch;

-

- /// if Find matches the label, this var stores the pattern that matched the

- /// query

- std::string mMatchedPattern;

-

- /// if Find matches the label, this var stores the the masked characters.

- /// The mask is given by '%' symbols

- std::string mMatchedPatternMask;

-

- /**

- * @brief Returns the directory depth of path

- */

- size_t

- DirDepth(const std::string & path);

-

-

- };

-

-

- /**

- * @brief MLF output buffer definition

- */

- template<

- typename _CharT,

- typename _Traits = std::char_traits<_CharT>,

- typename _CharTA = std::allocator<_CharT>,

- typename ByteT = char,

- typename ByteAT = std::allocator<ByteT>

- >

- class BasicOMlfStreamBuf

- : public std::basic_streambuf<_CharT, _Traits>

- {

- public:

- // necessary typedefs ....................................................

- typedef BasicOMlfStreamBuf<_CharT,_Traits,_CharTA,ByteT,ByteAT>

- this_type;

- typedef std::basic_ostream<_CharT, _Traits>&

- OStreamReference;

- typedef std::basic_streambuf<_CharT, _Traits>

- StreamBufType;

- typedef _CharTA char_allocator_type;

- typedef _CharT char_type;

- typedef typename _Traits::int_type int_type;

- typedef typename _Traits::pos_type pos_type;

- typedef ByteT byte_type;

- typedef ByteAT byte_allocator_type;

- typedef byte_type* byte_buffer_type;

- typedef std::vector<byte_type, byte_allocator_type > byte_vector_type;

- typedef std::vector<char_type, char_allocator_type > char_vector_type;

-

-

- BasicOMlfStreamBuf(OStreamReference rOStream, size_t bufferSize);

-

- ~BasicOMlfStreamBuf();

-

- // virtual functions inherited from basic_streambuf.......................

- int

- sync();

-

- /**

- * @brief Write character in the case of overflow

- * @param c Character to be written.

- * @return A value different than EOF (or traits::eof() for other traits)

- * signals success. If the function fails, either EOF

- * (or traits::eof() for other traits) is returned or an

- * exception is thrown.

- */

- int_type

- overflow(int_type c = _Traits::eof());

-

-

- // MLF specific functions ................................................

- /**

- * @brief Creates a new MLF block

- * @param rFileName filename to be opened

- */

- this_type*

- Open(const std::string& rFileName);

-

- /**

- * @brief Closes MLF block

- */

- void

- Close();

-

- /**

- * @brief Returns true if the MLF is now in open state

- */

- bool

- IsOpen() const

- { return mIsOpen; }

-

- LabelContainer&

- rLabels()

- { return mLabels; }

-

- private:

- bool mIsOpen;

- char_type mLastChar;

- OStreamReference mOStream;

- LabelContainer mLabels;

- }; // class BasicOMlfStreamBuf

-

-

-

- /**

- * @brief MLF input buffer definition

- */

- template<

- typename _CharT,

- typename _Traits = std::char_traits<_CharT>,

- typename _CharTA = std::allocator<_CharT>,

- typename ByteT = char,

- typename ByteAT = std::allocator<ByteT>

- >

- class BasicIMlfStreamBuf

- : public std::basic_streambuf<_CharT, _Traits>

- {

- private:

- // internal automaton states

- static const int IN_HEADER_STATE = 0;

- static const int OUT_OF_BODY_STATE = 1;

- static const int IN_TITLE_STATE = 2;

- static const int IN_BODY_STATE = 3;

-

-

- public: // necessary typedefs ..............................................

- typedef BasicIMlfStreamBuf<_CharT,_Traits,_CharTA,ByteT,ByteAT>

- this_type;

- typedef std::basic_istream<_CharT, _Traits>& IStreamReference;

- typedef std::basic_streambuf<_CharT, _Traits>

- StreamBufType;

- typedef _CharTA char_allocator_type;

- typedef _CharT char_type;

- typedef typename _Traits::int_type int_type;

- typedef typename _Traits::pos_type pos_type;

- typedef ByteT byte_type;

- typedef ByteAT byte_allocator_type;

- typedef byte_type* byte_buffer_type;

- typedef std::vector<byte_type, byte_allocator_type > byte_vector_type;

- typedef std::vector<char_type, char_allocator_type > char_vector_type;

-

-

- public:

- // constructors and destructors ..........................................

- BasicIMlfStreamBuf(IStreamReference rIStream, size_t bufferSize = 1024);

-

- ~BasicIMlfStreamBuf();

-

- // virtual functions inherited from basic_streambuf.......................

- /**

- * @brief Get character in the case of underflow

- *

- * @return The new character available at the get pointer position, if

- * any. Otherwise, traits::eof() is returned.

- */

- int_type

- underflow();

-

-

- // MLF specific functions ................................................

- /**

- * @brief Creates a new MLF block

- * @param rFileName filename to be opened

- */

- this_type*

- Open(const std::string& rFileName);

-

- /**

- * @brief Closes MLF block

- */

- this_type*

- Close();

-

- /**

- * @brief Returns true if the MLF is now in open state

- */

- bool

- IsOpen() const

- { return mIsOpen; }

-

- /**

- * @brief Parses the stream (if possible) and stores positions to the

- * label titles

- */

- void

- Index();

-

- bool

- IsHashed() const

- { return mIsHashed; }

-

- /**

- * @brief Jumps to next label definition

- * @param rName std::string to be filled with the label name

- * @return true on success

- *

- * The procedure automatically tries to hash the labels.

- */

- bool

- JumpToNextDefinition(std::string& rName);

-

- /**

- * @brief Returns reference to the base stream

- * @return reference to the stream

- *

- */

- IStreamReference

- GetBaseStream()

- {

- return mIStream;

- }

-

- private: // auxillary functions ............................................

- /**

- * @brief Fills the line buffer with next line and updates the internal

- * state of the finite automaton

- */

- void

- FillLineBuffer();

-

-

- private: // atributes ......................................................

- // some flags

- bool mIsOpen;

- bool mIsHashed;

- bool mIsEof;

-

- /// internal state of the finite automaton

- int mState;

-

- IStreamReference mIStream;

- LabelContainer mLabels;

-

- std::vector<char_type> mLineBuffer;

- }; // class BasicIMlfStreamBuf

-

-

-

-

- /**

- * @brief Base class with type-independent members for the Mlf Output

- * Stram class

- *

- * This is a derivative of the basic_ios class. We derive it as we need

- * to override some member functions

- */

- template<

- typename Elem,

- typename Tr = std::char_traits<Elem>,

- typename ElemA = std::allocator<Elem>,

- typename ByteT = char,

- typename ByteAT = std::allocator<ByteT>

- >

- class BasicOMlfStreamBase

- : virtual public std::basic_ios<Elem,Tr>

- {

- public:

- typedef std::basic_ostream<Elem, Tr>& OStreamReference;

- typedef BasicOMlfStreamBuf <

- Elem,Tr,ElemA,ByteT,ByteAT> OMlfStreamBufType;

-

- /**

- * @brief constructor

- *

- * @param rOStream user defined output stream

- */

- BasicOMlfStreamBase(OStreamReference rOStream,

- size_t bufferSize)

- : mBuf(rOStream, bufferSize)

- { this->init(&mBuf); };

-

- /**

- * @brief Returns a pointer to the buffer object for this stream

- */

- OMlfStreamBufType*

- rdbuf()

- { return &mBuf; };

-

- private:

- OMlfStreamBufType mBuf;

- };

-

-

- template<

- typename Elem,

- typename Tr = std::char_traits<Elem>,

- typename ElemA = std::allocator<Elem>,

- typename ByteT = char,

- typename ByteAT = std::allocator<ByteT>

- >

- class BasicIMlfStreamBase

- : virtual public std::basic_ios<Elem,Tr>

- {

- public:

- typedef std::basic_istream<Elem, Tr>& IStreamReference;

- typedef BasicIMlfStreamBuf <

- Elem,Tr,ElemA,ByteT,ByteAT> IMlfStreamBufType;

-

- BasicIMlfStreamBase( IStreamReference rIStream,

- size_t bufferSize)

- : mBuf(rIStream, bufferSize)

- { this->init(&mBuf ); };

-

- IMlfStreamBufType*

- rdbuf()

- { return &mBuf; };

-

- IStreamReference

- GetBaseStream()

- { return mBuf.GetBaseStream(); }

-

- private:

- IMlfStreamBufType mBuf;

- };

-

-

- template<

- typename Elem,

- typename Tr = std::char_traits<Elem>,

- typename ElemA = std::allocator<Elem>,

- typename ByteT = char,

- typename ByteAT = std::allocator<ByteT>

- >

- class BasicOMlfStream

- : public BasicOMlfStreamBase<Elem,Tr,ElemA,ByteT,ByteAT>,

- public std::basic_ostream<Elem,Tr>

- {

- public:

- typedef BasicOMlfStreamBase< Elem,Tr,ElemA,ByteT,ByteAT>

- BasicOMlfStreamBaseType;

- typedef std::basic_ostream<Elem,Tr> OStreamType;

- typedef OStreamType& OStreamReference;

-

- BasicOMlfStream(OStreamReference rOStream, size_t bufferSize = 32)

- : BasicOMlfStreamBaseType(rOStream, bufferSize),

- OStreamType(BasicOMlfStreamBaseType::rdbuf())

- { }

-

- /**

- * @brief Destructor closes the stream

- */

- ~BasicOMlfStream()

- { }

-

-

- /**

- * @brief Creates a new MLF block

- * @param rFileName filename to be opened

- */

- void

- Open(const std::string& rFileName)

- { BasicOMlfStreamBaseType::rdbuf()->Open(rFileName); }

-

- /**

- * @brief Closes MLF block

- */

- void

- Close()

- { BasicOMlfStreamBaseType::rdbuf()->Close(); }

-

- /**

- * @brief Returns true if the MLF is now in open state

- */

- bool

- IsOpen() const

- { return BasicOMlfStreamBaseType::rdbuf()->IsOpen(); }

-

- /**

- * @brief Accessor to the label container

- * @return Reference to the label container

- */

- LabelContainer&

- rLabels()

- { return BasicOMlfStreamBaseType::rdbuf()->rLabels(); }

- };

-

-

-

- template<

- typename Elem,

- typename Tr = std::char_traits<Elem>,

- typename ElemA = std::allocator<Elem>,

- typename ByteT = char,

- typename ByteAT = std::allocator<ByteT>

- >

- class BasicIMlfStream

- : public BasicIMlfStreamBase<Elem,Tr,ElemA,ByteT,ByteAT>,

- public std::basic_istream<Elem,Tr>

- {

- public:

- typedef BasicIMlfStreamBase <Elem,Tr,ElemA,ByteT,ByteAT>

- BasicIMlfStreamBaseType;

- typedef std::basic_istream<Elem,Tr> IStreamType;

- typedef IStreamType& IStreamReference;

- typedef unsigned char byte_type;

-

- BasicIMlfStream(IStreamReference rIStream, size_t bufferSize = 32)

- : BasicIMlfStreamBaseType(rIStream, bufferSize),

- IStreamType(BasicIMlfStreamBaseType::rdbuf())

- {};

-

-

- /**

- * @brief Creates a new MLF block

- * @param rFileName filename to be opened

- */

- void

- Open(const std::string& rFileName)

- {

- std::basic_streambuf<Elem, Tr>* p_buf;

-

- p_buf = BasicIMlfStreamBaseType::rdbuf()->Open(rFileName);

-

- if (NULL == p_buf) {

- IStreamType::clear(IStreamType::rdstate() | std::ios::failbit);

- }

- else {

- IStreamType::clear();

- }

- }

-

- /**

- * @brief Closes MLF block.

- * In fact, nothing is done

- */

- void

- Close()

- {

- if (NULL == BasicIMlfStreamBaseType::rdbuf()->Close()) {

- IStreamType::clear(IStreamType::rdstate() | std::ios::failbit);

- }

- }

-

- void

- Index()

- { BasicIMlfStreamBaseType::rdbuf()->Index(); }

-

- bool

- IsHashed() const

- { return BasicIMlfStreamBaseType::rdbuf()->IsHashed(); }

-

- };

-

-

-

- // MAIN TYPEDEFS..............................................................

- typedef BasicOMlfStream<char> OMlfStream;

- typedef BasicOMlfStream<wchar_t> WOMlfStream;

- typedef BasicIMlfStream<char> IMlfStream;

- typedef BasicIMlfStream<wchar_t> WIMlfStream;

-

-

-#ifdef PATH_MAX

- const size_t MAX_LABEL_DEPTH = PATH_MAX;

-#else

- const size_t MAX_LABEL_DEPTH = 1024;

-#endif

-

-

-} // namespace TNet

-

-#include "MlfStream.tcc"

-

-#endif

-#ifndef STK_MlfStream_tcc

-#define STK_MlfStream_tcc

-

-#include <algorithm>

-

-#include "Common.h"

-#include "StkMatch.h"

-

-namespace TNet

-{

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- BasicOMlfStreamBuf(OStreamReference rOStream, size_t bufferSize)

- : mIsOpen(false), mOStream(rOStream)

- { }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- ~BasicOMlfStreamBuf()

- {

- mOStream.flush();

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- int

- BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- sync()

- {

- mOStream.flush();

- return 0;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- typename _Traits::int_type

- BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- overflow(typename _Traits::int_type c)

- {

- // we don't use buffer here...

- if (mIsOpen) {

- if (_Traits::eof() == c) {

- return _Traits::not_eof(c);

- }

- // only pass the character to the stream

- mOStream.rdbuf()->sputc(c);

-

- // remember last char (in case we want to close)

- mLastChar = c;

-

- return c;

- }

- else {

- return _Traits::eof();

- }

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- void

- BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- Close()

- {

- // if last character was not EOL, we need to insert it

- if (mLastChar != '\n') {

- mOStream.put('\n');

- }

- mOStream << ".\n";

-

- // flush the stream and declare the stream closed

- mOStream.flush();

- mIsOpen = false;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT> *

- BasicOMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- Open(const std::string& rFileName)

- {

- // retreive position

- std::streampos pos = mOStream.tellp();

-

- // write the initial "filename" in parantheses

- mOStream << '"' << rFileName << '"' << std::endl;

- mLastChar = '\n';

-

- // return NULL if we canot open

- if (!mOStream.good()) {

- return NULL;

- }

-

- // if ok, store the name position

- if (-1 != pos) {

- pos = mOStream.tellp();

- mLabels.Insert(rFileName, pos);

- }

-

- // set open flag and return this

- mIsOpen = true;

- return this;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- // BasicIMlfStreamBuf section

- //

- //****************************************************************************

- //****************************************************************************

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- BasicIMlfStreamBuf(IStreamReference rIStream, size_t bufferSize)

- : mIsOpen(false), mIsHashed(false), mIsEof(true), mState(IN_HEADER_STATE),

- mIStream(rIStream), mLineBuffer()

- {

- // we reserve some place for the buffer...

- mLineBuffer.reserve(bufferSize);

-

- //StreamBufType::setg(mpBuffer, mpBuffer + bufferSize, mpBuffer + bufferSize);

- StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.back()), &(mLineBuffer.back()));

- }

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- ~BasicIMlfStreamBuf()

- {

- }

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- void

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- Index()

- {

- // retreive position

- std::streampos orig_pos = mIStream.tellg();

- int orig_state = mState;

-

- // for streams like stdin, pos will by definition be -1, so we can only

- // rely on sequential access and cannot hash it.

- if (-1 != orig_pos) {

- std::string aux_name;

- // we will constantly jump to next definition. the function automatically

- // hashes the stream if possible

- while (JumpToNextDefinition(aux_name))

- { }

-

- // move to the original position

- mIStream.clear();

- mIStream.seekg(orig_pos);

- mState = orig_state;

-

- // set as hashed

- mIsHashed=true;

- }

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- bool

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- JumpToNextDefinition(std::string& rName)

- {

- if (!mIStream.good()) {

- return false;

- }

-

- // if we can, we will try to index the label

- std::streampos pos = mIStream.tellg();

-

- // we might be at a definition already, so first move one line further

- FillLineBuffer();

-

- // read lines till we get to definition again

- while (mIStream.good() && mState != IN_TITLE_STATE) {

- FillLineBuffer();

- }

-

- // decide what happened

- if (IN_TITLE_STATE == mState) {

- // if we can, we will try to index the label

- pos = mIStream.tellg();

-

- if (pos != static_cast<const std::streampos>(-1)) {

- // if (pos !=std::string::npos) { // This line does not work under MinGW

- std::string line_buffer(mLineBuffer.begin(), mLineBuffer.end());

- TNet::ParseHTKString(line_buffer, rName);

- mLabels.Insert(rName, pos);

- }

-

- return true;

- }

- else {

- // we have been hashing all the way through so we know that if this is

- // is the EOF, we are done hashing this stream

- if (pos != static_cast<const std::streampos>(-1)) {

- mIsHashed = true;

- }

-

- // we are not in body state, so we just return false

- return false;

- }

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>*

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- Close()

- {

- if (!mIsOpen) {

- mIsEof = true;

- return NULL;

- }

- else {

- // if we try to close while in the body, we need to reach the end

- if (mState == IN_BODY_STATE) {

- while (mState == IN_BODY_STATE) {

- FillLineBuffer();

- }

- }

-

- // disable buffer mechanism

- StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()),

- &(mLineBuffer.front()));

-

- mIsEof = true;

- mIsOpen = false;

-

- return this;

- }

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>*

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- Open(const std::string& rFileName)

- {

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>* ret_val = NULL;

-

- // this behavior is compatible with ifstream

- if (mIsOpen) {

- Close();

- return NULL;

- }

-

- // retreive position

- std::streampos pos = mIStream.tellg();

- LabelRecord label_record;

-

- // for streams like stdin, pos will by definition be -1, so we can only

- // rely on sequential access. At this place, we decide what to do

- if ((-1 != pos) && (mLabels.Find(rFileName, label_record))) {

- mIStream.seekg(label_record.mStreamPos);

- mState = IN_TITLE_STATE;

-

- // we don't want the other stream to be bad, so we transfer the

- // flagbits to this stream

- if (!mIStream.good()) {

- mIStream.clear();

- mIsOpen = false;

- ret_val = NULL;

- }

- else {

- mIsOpen = true;

- mIsEof = false;

- ret_val = this;

- }

- }

-

- // we don't have sequential stream and we didn't find the label, but

- // we are hashed, so we can be sure, that we failed

- else if ((-1 != pos) && mIsHashed) {

- mIsOpen = false;

- ret_val = NULL;

- }

-

- // we either have sequential stream or didn't find anything, but we can

- // still try to sequentially go and look for it

- else {

- bool found = false;

- std::string aux_name;

- std::string aux_name2;

-

- while ((!found) && JumpToNextDefinition(aux_name)) {

- if (TNet::ProcessMask(rFileName, aux_name, aux_name2)) {

- mIsOpen = true;

- mIsEof = false;

- found = true;

- ret_val = this;

- }

- }

-

- if (!found) {

- mIsOpen = false;

- ret_val = NULL;

- }

- }

-

- return ret_val;

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- typename _Traits::int_type

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- underflow()

- {

- // we don't do anything if EOF

- if (mIsEof) {

- StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()),

- &(mLineBuffer.front()));

- return _Traits::eof();

- }

-

- // read from buffer if we can

- if (StreamBufType::gptr() && (StreamBufType::gptr() < StreamBufType::egptr())) {

- return _Traits::not_eof(*StreamBufType::gptr());

- }

-

- // might happen that stream is in !good state

- if (!mIStream.good()) {

- mIsEof = true;

- StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()),

- &(mLineBuffer.front()));

- return _Traits::eof();

- }

-

- // fill the line buffer and update my state

- FillLineBuffer();

-

- // if the whole line is just period or it's eof, declare EOF

- if (mState == OUT_OF_BODY_STATE) {

- mIsEof = true;

- StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()),

- &(mLineBuffer.front()));

- return _Traits::eof();

- }

-

- // restore the buffer mechanism

- StreamBufType::setg(&(mLineBuffer.front()), &(mLineBuffer.front()),

- &(mLineBuffer.back()) + 1);

-

- return *StreamBufType::gptr();

- }

-

-

- //****************************************************************************

- //****************************************************************************

- template<

- typename _CharT,

- typename _Traits,

- typename _CharTA,

- typename ByteT,

- typename ByteAT

- >

- void

- BasicIMlfStreamBuf<_CharT, _Traits, _CharTA, ByteT, ByteAT>::

- FillLineBuffer()

- {

- // reset line buffer

- size_t capacity = mLineBuffer.capacity();

- mLineBuffer.clear();

- mLineBuffer.reserve(capacity);

-

- // read one line into buffer

- int c;

- while ((c = mIStream.get()) != '\n' && c != _Traits::eof()) {

- mLineBuffer.push_back(char(c));

- }

-

- // we want to be able to pass last eol symbol

- if (c == '\n') {

- mLineBuffer.push_back(char(c));

- }

-

- // we will decide where we are

- switch (mState) {

- case IN_HEADER_STATE:

-

- case OUT_OF_BODY_STATE:

- if (mLineBuffer[0] != '#') {

- mState = IN_TITLE_STATE;

- }

- break;

-

- case IN_TITLE_STATE:

- if (mLineBuffer[0] == '.' && (mLineBuffer.back() == '\n' || mIStream.eof())) {

- mState = OUT_OF_BODY_STATE;

- }

- else {

- mState = IN_BODY_STATE;

- }

- break;

-

- case IN_BODY_STATE:

- // period or EOF will end the file

- if (mLineBuffer[0] == '.' && (mLineBuffer.back() == '\n' || mIStream.eof())) {

- mState = OUT_OF_BODY_STATE;

- }

- if (mLineBuffer.size() == 0) {

- mState = OUT_OF_BODY_STATE;

- }

- break;

- }

- }

-} // namespace TNet

-

-

-#endif // STK_MlfStream_tcc

-/*

- EPSHeader

-

- File: filmatch.c

- Author: J. Kercheval

- Created: Thu, 03/14/1991 22:22:01

-*/

-

-/*

- EPSRevision History

- O. Glembek Thu, 03/11/2005 01:58:00 Added Mask extraction support (char % does this)

- J. Kercheval Wed, 02/20/1991 22:29:01 Released to Public Domain

- J. Kercheval Fri, 02/22/1991 15:29:01 fix '\' bugs (two :( of them)

- J. Kercheval Sun, 03/10/1991 19:31:29 add error return to matche()

- J. Kercheval Sun, 03/10/1991 20:11:11 add is_valid_pattern code

- J. Kercheval Sun, 03/10/1991 20:37:11 beef up main()

- J. Kercheval Tue, 03/12/1991 22:25:10 Released as V1.1 to Public Domain

- J. Kercheval Thu, 03/14/1991 22:22:25 remove '\' for DOS file parsing

- J. Kercheval Thu, 03/28/1991 20:58:27 include filmatch.h

-*/

-

-/*

- Wildcard Pattern Matching

-*/

-

-

-#include "StkMatch.h"

-#include "Common.h"

-

-namespace TNet

-{

- //#define TEST

- static int matche_after_star (register const char *pattern, register const char *text, register char *s);

- // following function is not defined or used.

- // static int fast_match_after_star (register const char *pattern, register const char *text);

-

- /*----------------------------------------------------------------------------

- *

- * Return true if PATTERN has any special wildcard characters

- *

- ----------------------------------------------------------------------------*/

-

- bool is_pattern (const char *p)

- {

- while ( *p ) {

- switch ( *p++ ) {

- case '?':

- case '*':

- case '%':

- case '[':

- return true;

- }

- }

- return false;

- }

-

-

- /*----------------------------------------------------------------------------

- *

- * Return true if PATTERN has is a well formed regular expression according

- * to the above syntax

- *

- * error_type is a return code based on the type of pattern error. Zero is

- * returned in error_type if the pattern is a valid one. error_type return

- * values are as follows:

- *

- * PATTERN_VALID - pattern is well formed

- * PATTERN_RANGE - [..] construct has a no end range in a '-' pair (ie [a-])

- * PATTERN_CLOSE - [..] construct has no end bracket (ie [abc-g )

- * PATTERN_EMPTY - [..] construct is empty (ie [])

- *

- ----------------------------------------------------------------------------*/

-

- bool is_valid_pattern (const char *p, int *error_type)

- {

-

- /* init error_type */

- *error_type = PATTERN_VALID;

-

- /* loop through pattern to EOS */

- while ( *p )

- {

- /* determine pattern type */

- switch ( *p )

- {

- /* the [..] construct must be well formed */

- case '[':

- {

- p++;

-

- /* if the next character is ']' then bad pattern */

- if ( *p == ']' ) {

- *error_type = PATTERN_EMPTY;

- return false;

- }

-

- /* if end of pattern here then bad pattern */

- if ( !*p )

- {

- *error_type = PATTERN_CLOSE;

- return false;

- }

-

- /* loop to end of [..] construct */

- while ( *p != ']' )

- {

- /* check for literal escape */

- if ( *p == '\\' )

- {

- p++;

-

- /* if end of pattern here then bad pattern */

- if ( !*p++ ) {

- *error_type = PATTERN_ESC;

- return false;

- }

- }

- else

- p++;

-

- /* if end of pattern here then bad pattern */

- if ( !*p )

- {

- *error_type = PATTERN_CLOSE;

- return false;

- }

-

- /* if this a range */

- if ( *p == '-' )

- {

- /* we must have an end of range */

- if ( !*++p || *p == ']' )

- {

- *error_type = PATTERN_RANGE;

- return false;

- }

- else

- {

-

- /* check for literal escape */

- if ( *p == '\\' )

- p++;

-

- /* if end of pattern here then bad pattern */

- if ( !*p++ )

- {

- *error_type = PATTERN_ESC;

- return false;

- }

- }

- }

- }

- break;

- } //case '[':

-

-

- /* all other characters are valid pattern elements */

- case '*':

- case '?':

- case '%':

- default:

- p++; /* "normal" character */

- break;

- } // switch ( *p )

- } // while ( *p )

-

- return true;

- } //bool is_valid_pattern (const char *p, int *error_type)

-

-

- /*----------------------------------------------------------------------------

- *

- * Match the pattern PATTERN against the string TEXT;

- *

- * returns MATCH_VALID if pattern matches, or an errorcode as follows

- * otherwise:

- *

- * MATCH_PATTERN - bad pattern

- * MATCH_RANGE - match failure on [..] construct

- * MATCH_ABORT - premature end of text string

- * MATCH_END - premature end of pattern string

- * MATCH_VALID - valid match

- *

- *

- * A match means the entire string TEXT is used up in matching.

- *

- * In the pattern string:

- * `*' matches any sequence of characters (zero or more)

- * `?' matches any character

- * `%' matches any character and stores it in the s string

- * [SET] matches any character in the specified set,

- * [!SET] or [^SET] matches any character not in the specified set.

- * \ is allowed within a set to escape a character like ']' or '-'

- *

- * A set is composed of characters or ranges; a range looks like

- * character hyphen character (as in 0-9 or A-Z). [0-9a-zA-Z_] is the

- * minimal set of characters allowed in the [..] pattern construct.

- * Other characters are allowed (ie. 8 bit characters) if your system

- * will support them.

- *

- * To suppress the special syntactic significance of any of `[]*?%!^-\',

- * within a [..] construct and match the character exactly, precede it

- * with a `\'.

- *

- ----------------------------------------------------------------------------*/

-

- int matche ( register const char *p, register const char *t, register char *s )

- {

- register char range_start, range_end; /* start and end in range */

-

- bool invert; /* is this [..] or [!..] */

- bool member_match; /* have I matched the [..] construct? */

- bool loop; /* should I terminate? */

-

- for ( ; *p; p++, t++ ) {

-

- /* if this is the end of the text then this is the end of the match */

- if (!*t) {

- return ( *p == '*' && *++p == '\0' ) ? MATCH_VALID : MATCH_ABORT;

- }

-

- /* determine and react to pattern type */

- switch ( *p ) {

-

- /* single any character match */

- case '?':

- break;

-

- /* single any character match, with extraction*/

- case '%': {

- *s++ = *t;

- *s = '\0';

- break;

- }

-

- /* multiple any character match */

- case '*':

- return matche_after_star (p, t, s);

-

- /* [..] construct, single member/exclusion character match */

- case '[': {

- /* move to beginning of range */

- p++;

-

- /* check if this is a member match or exclusion match */

- invert = false;

- if ( *p == '!' || *p == '^') {

- invert = true;

- p++;

- }

-

- /* if closing bracket here or at range start then we have a

- malformed pattern */

- if ( *p == ']' ) {

- return MATCH_PATTERN;

- }

-

- member_match = false;

- loop = true;

-

- while ( loop ) {

-

- /* if end of construct then loop is done */

- if (*p == ']') {

- loop = false;

- continue;

- }

-

- /* matching a '!', '^', '-', '\' or a ']' */

- if ( *p == '\\' ) {

- range_start = range_end = *++p;