summaryrefslogtreecommitdiff
path: root/tnet_io/KaldiLib/Features.cc
diff options
context:
space:
mode:
authorDeterminant <[email protected]>2015-06-25 12:56:45 +0800
committerDeterminant <[email protected]>2015-06-25 12:56:45 +0800
commita74183ddb4ab8383bfe214b3745eb8a0a99ee47a (patch)
treed5e69cf8c4c2db2e3a4722778352fc3c95953bb2 /tnet_io/KaldiLib/Features.cc
parentb6301089cde20f4c825c7f5deaf179082aad63da (diff)
let HTK I/O implementation be a single package
Diffstat (limited to 'tnet_io/KaldiLib/Features.cc')
-rw-r--r--tnet_io/KaldiLib/Features.cc1798
1 files changed, 0 insertions, 1798 deletions
diff --git a/tnet_io/KaldiLib/Features.cc b/tnet_io/KaldiLib/Features.cc
deleted file mode 100644
index 64b63e8..0000000
--- a/tnet_io/KaldiLib/Features.cc
+++ /dev/null
@@ -1,1798 +0,0 @@
-
-//enable feature repository profiling
-#define PROFILING 1
-
-#include <sstream>
-#include <map>
-#include <list>
-#include <cstdio>
-
-#include "Features.h"
-#include "Tokenizer.h"
-#include "StkMatch.h"
-#include "Types.h"
-
-
-
-namespace TNet
-{
- const char
- FeatureRepository::
- mpParmKindNames[13][16] =
- {
- {"WAVEFORM"},
- {"LPC"},
- {"LPREFC"},
- {"LPCEPSTRA"},
- {"LPDELCEP"},
- {"IREFC"},
- {"MFCC"},
- {"FBANK"},
- {"MELSPEC"},
- {"USER"},
- {"DISCRETE"},
- {"PLP"},
- {"ANON"}
- };
-
- //***************************************************************************
- //***************************************************************************
-
- FileListElem::
- FileListElem(const std::string & rFileName)
- {
- std::string::size_type pos;
-
- mLogical = rFileName;
- mWeight = 1.0;
-
- // some slash-backslash replacement hack
- for (size_t i = 0; i < mLogical.size(); i++) {
- if (mLogical[i] == '\\') {
- mLogical[i] = '/';
- }
- }
-
- // read sentence weight definition if any ( physical_file.fea[s,e]{weight} )
- if ((pos = mLogical.find('{')) != std::string::npos)
- {
- std::string tmp_weight(mLogical.begin() + pos + 1, mLogical.end());
- std::stringstream tmp_ss(tmp_weight);
-
- tmp_ss >> mWeight;
- mLogical.erase(pos);
- }
-
- // look for "=" symbol and if found, split it
- if ((pos = mLogical.find('=')) != std::string::npos)
- {
- // copy all from mLogical[pos+1] till the end to mPhysical
- mPhysical.assign(mLogical.begin() + pos + 1, mLogical.end());
- // erase all from pos + 1 till the end from mLogical
- mLogical.erase(pos);
- // trim the leading and trailing spaces
- Trim(mPhysical);
- Trim(mLogical);
- }
- else
- {
- // trim the leading and trailing spaces
- Trim(mLogical);
-
- mPhysical = mLogical;
- }
- }
-
-
- //###########################################################################
- //###########################################################################
- // FeatureRepository section
- //###########################################################################
- //###########################################################################
-
- //***************************************************************************
- //***************************************************************************
- void
- FeatureRepository::
- ReadCepsNormFile(
- const char * pFileName,
- char ** pLastFileName,
- BaseFloat ** vec_buff,
- int sampleKind,
- CNFileType type,
- int coefs)
- {
- FILE* fp;
- int i;
- char s1[64];
- char s2[64];
- const char* typeStr = (type == CNF_Mean ? "MEAN" :
- type == CNF_Variance ? "VARIANCE" : "VARSCALE");
-
- const char* typeStr2 = (type == CNF_Mean ? "CMN" :
- type == CNF_Variance ? "CVN" : "VarScale");
-
- if (*pLastFileName != NULL && !strcmp(*pLastFileName, pFileName)) {
- return;
- }
- free(*pLastFileName);
- *pLastFileName=strdup(pFileName);
- *vec_buff = (BaseFloat*) realloc(*vec_buff, coefs * sizeof(BaseFloat));
-
- if (*pLastFileName == NULL || *vec_buff== NULL)
- throw std::runtime_error("Insufficient memory");
-
- if ((fp = fopen(pFileName, "r")) == NULL) {
- throw std::runtime_error(std::string("Cannot open ") + typeStr2
- + " pFileName: '" + pFileName + "'");
- }
-
- if ((type != CNF_VarScale
- && (fscanf(fp, " <%64[^>]> <%64[^>]>", s1, s2) != 2
- || strcmp(StrToUpper(s1), "CEPSNORM")
- || ReadParmKind(s2, false) != sampleKind))
- || fscanf(fp, " <%64[^>]> %d", s1, &i) != 2
- || strcmp(StrToUpper(s1), typeStr)
- || i != coefs)
- {
- ParmKind2Str(sampleKind, s2);
-
- //std::cout << "[[[TADY!!!!]]]" << pFileName << "\n" << std::flush;
-
- throw std::runtime_error(std::string("")
- + (type == CNF_VarScale ? "" : "<CEPSNORM> <")
- + (type == CNF_VarScale ? "" : s2)
- + (type == CNF_VarScale ? "" : ">")
- + " <" + typeStr + " ... expected in " + typeStr2
- + " file " + pFileName);
- }
-
- for (i = 0; i < coefs; i++) {
- if (fscanf(fp, " "FLOAT_FMT, *vec_buff+i) != 1) {
- if (fscanf(fp, "%64s", s2) == 1) {
- throw std::runtime_error(std::string("Decimal number expected but '")
- + s2 + "' found in " + typeStr2 + " file " + pFileName);
- }
- else if (feof(fp)) {
- throw std::runtime_error(std::string("Unexpected end of ")
- + typeStr2 + " file "+ pFileName);
- }
- else {
- throw std::runtime_error(std::string("Cannot read ") + typeStr2
- + " file " + pFileName);
- }
- }
-
- if (type == CNF_Variance)
- (*vec_buff)[i] = BaseFloat(1 / sqrt((*vec_buff)[i]));
- else if (type == CNF_VarScale)
- (*vec_buff)[i] = BaseFloat(sqrt((*vec_buff)[i]));
- }
-
- if (fscanf(fp, "%64s", s2) == 1)
- {
- throw std::runtime_error(std::string("End of file expected but '")
- + s2 + "' found in " + typeStr2 + " file " + pFileName);
- }
-
- fclose(fp);
- } // ReadCepsNormFile(...)
-
-
- //***************************************************************************
- //***************************************************************************
- void
- FeatureRepository::
- HtkFilter(const char* pFilter, const char* pValue, FeatureRepository& rOut)
- {
- std::list<FileListElem>::iterator it;
- std::string str;
-
- rOut.mSwapFeatures = mSwapFeatures;
- rOut.mStartFrameExt = mStartFrameExt;
- rOut.mEndFrameExt = mEndFrameExt;
- rOut.mTargetKind = mTargetKind;
- rOut.mDerivOrder = mDerivOrder;
- rOut.mDerivWinLengths = mDerivWinLengths;
-
- rOut.mpCvgFile = mpCvgFile;
- rOut.mpCmnPath = mpCmnPath;
- rOut.mpCmnMask = mpCmnMask;
- rOut.mpCvnPath = mpCvnPath;
- rOut.mpCvnMask = mpCvnMask;
-
- rOut.mInputQueue.clear();
-
- // go through all records and check the mask
- for (it=mInputQueue.begin(); it!= mInputQueue.end(); ++it) {
- if (pFilter == NULL
- || (ProcessMask(it->Logical(), pFilter, str) && (str == pValue))) {
- rOut.mInputQueue.push_back(*it);
- }
- }
-
- // set the queue position to the begining
- rOut.mInputQueueIterator = mInputQueue.end();
-
- rOut.mCurrentIndexFileName = "";
- rOut.mCurrentIndexFileDir = "";
- rOut.mCurrentIndexFileExt = "";
-
- mStream.close();
- mStream.clear();
-
- rOut.mpLastFileName = NULL;
- rOut.mLastFileName = "";
- rOut.mpLastCmnFile = NULL;
- rOut.mpLastCvnFile = NULL;
- rOut.mpLastCvgFile = NULL;
- rOut.mpCmn = NULL;
- rOut.mpCvn = NULL;
- rOut.mpCvg = NULL;
- rOut.mpA = NULL;
- rOut.mpB = NULL;
-
- }
-
-
- //***************************************************************************
- //***************************************************************************
- void
- FeatureRepository::
- HtkSelection(const char* pFilter, std::list< std::string >& rOut)
- {
- std::map< std::string, bool> aux_map;
- std::map< std::string, bool>::iterator map_it;
- std::list<FileListElem>::iterator it;
- std::string str;
-
- rOut.clear();
-
- if(pFilter != NULL) {
- // go through all records and check the mask
- for (it=mInputQueue.begin(); it!= mInputQueue.end(); ++it) {
- if (ProcessMask(it->Logical(), pFilter, str)) {
- aux_map[str] = true;
- }
- }
- } else {
- aux_map[std::string("default speaker")] = true;
- }
-
- for (map_it = aux_map.begin(); map_it != aux_map.end(); ++map_it) {
- rOut.push_back(map_it->first);
- }
- }
-
-
- //***************************************************************************
- //***************************************************************************
- int
- FeatureRepository::
- ParmKind2Str(unsigned parmKind, char *pOutString)
- {
- // :KLUDGE: Absolutely no idea what this is...
- if ((parmKind & 0x003F) >= sizeof(mpParmKindNames)/sizeof(mpParmKindNames[0]))
- return 0;
-
- strcpy(pOutString, mpParmKindNames[parmKind & 0x003F]);
-
- if (parmKind & PARAMKIND_E) strcat(pOutString, "_E");
- if (parmKind & PARAMKIND_N) strcat(pOutString, "_N");
- if (parmKind & PARAMKIND_D) strcat(pOutString, "_D");
- if (parmKind & PARAMKIND_A) strcat(pOutString, "_A");
- if (parmKind & PARAMKIND_C) strcat(pOutString, "_C");
- if (parmKind & PARAMKIND_Z) strcat(pOutString, "_Z");
- if (parmKind & PARAMKIND_K) strcat(pOutString, "_K");
- if (parmKind & PARAMKIND_0) strcat(pOutString, "_0");
- if (parmKind & PARAMKIND_V) strcat(pOutString, "_V");
- if (parmKind & PARAMKIND_T) strcat(pOutString, "_T");
-
- return 1;
- }
-
-
- // //***************************************************************************
- // //***************************************************************************
- // void
- // AddFileListToFeatureRepositories(
- // const char* pFileName,
- // const char* pFilter,
- // std::queue<FeatureRepository *> &featureRepositoryList)
- // {
- // IStkStream l_stream;
- // std::string file_name;
- // Tokenizer file_list(pFileName, ",");
- // Tokenizer::iterator p_file_name;
-
- // //:TODO: error if empty featureRepositoryList
- //
- // for (p_file_name = file_list.begin(); p_file_name != file_list.end(); ++p_file_name)
- // {
- // // get rid of initial and trailing blanks
- // Trim(*p_file_name);
-
- // // open file name
- // l_stream.open(p_file_name->c_str(), std::ios::in, pFilter);
- //
- // if (!l_stream.good()) {
- // //:TODO:
- // // Warning or error ... Why warning? -Lukas
- // throw std::runtime_error(std::string("Cannot not open list file ") +
- // *p_file_name);
- // }
-
- // // read all lines and parse them
- // for(;;)
- // {
- // l_stream >> file_name;
- // //:TODO: if(l_stream.badl()) Error()
- // // Reading after last token set the fail bit
- // if(l_stream.fail())
- // break;
- // // we can push_back a std::string as new FileListElem object
- // // is created using FileListElem(const std::string&) constructor
- // // and logical and physical names are correctly extracted
- // featureRepositoryList.front()->mInputQueue.push_back(file_name);
- //
- // //cycle in the featureRepositoryList
- // featureRepositoryList.push(featureRepositoryList.front());
- // featureRepositoryList.pop();
- // }
- // l_stream.close();
- // }
- // } // AddFileList(const std::string & rFileName)
-
-
- //***************************************************************************
- //***************************************************************************
- void
- FeatureRepository::
- Init(
- bool swap,
- int extLeft,
- int extRight,
- int targetKind,
- int derivOrder,
- int* pDerivWinLen,
- const char* pCmnPath,
- const char* pCmnMask,
- const char* pCvnPath,
- const char* pCvnMask,
- const char* pCvgFile)
- {
- mSwapFeatures = swap;
- mStartFrameExt = extLeft;
- mEndFrameExt = extRight;
- mTargetKind = targetKind;
- mDerivOrder = derivOrder;
- mDerivWinLengths = pDerivWinLen;
- mpCmnPath = pCmnPath;
- mpCmnMask = pCmnMask;
- mpCvnPath = pCvnPath;
- mpCvnMask = pCvnMask;
- mpCvgFile = pCvgFile;
- } // Init()
-
-
- //***************************************************************************
- //***************************************************************************
- void
- FeatureRepository::
- AddFile(const std::string & rFileName)
- {
- mInputQueue.push_back(rFileName);
- } // AddFile(const std::string & rFileName)
-
-
- //***************************************************************************
- //***************************************************************************
- void
- FeatureRepository::
- AddFileList(const char* pFileName, const char* pFilter)
- {
- IStkStream l_stream;
- std::string file_name;
- Tokenizer file_list(pFileName, ",");
- Tokenizer::iterator p_file_name;
-
- for (p_file_name = file_list.begin(); p_file_name != file_list.end(); ++p_file_name)
- {
- // get rid of spaces
- Trim(*p_file_name);
-
- // open the file
- l_stream.open(p_file_name->c_str(), std::ios::in, pFilter);
-
- if (!l_stream.good())
- {
- //:TODO:
- // Warning or error ... Why warning? -Lukas
- throw std::runtime_error(std::string("Cannot not open list file ") +
- *p_file_name);
- }
- // read all lines and parse them
- for(;;)
- {
- l_stream >> file_name;
- //:TODO: if(l_stream.badl()) Error()
- // Reading after last token set the fail bit
- if(l_stream.fail())
- break;
- // we can push_back a std::string as new FileListElem object
- // is created using FileListElem(const std::string&) constructor
- // and logical and physical names are correctly extracted
- mInputQueue.push_back(file_name);
- }
- l_stream.close();
- }
- } // AddFileList(const std::string & rFileName)
-
-
- //***************************************************************************
- //***************************************************************************
- void
- FeatureRepository::
- MoveNext()
- {
- assert (mInputQueueIterator != mInputQueue.end());
- mInputQueueIterator++;
- } // ReadFullMatrix(Matrix<BaseFloat>& rMatrix)
-
-
- //***************************************************************************
- //***************************************************************************
- bool
- FeatureRepository::
- ReadFullMatrix(Matrix<BaseFloat>& rMatrix)
- {
- // clear the matrix
- rMatrix.Destroy();
-
- // extract index file name
- if (!mCurrentIndexFileDir.empty())
- {
- char tmp_name[mCurrentIndexFileDir.length() +
- mCurrentIndexFileExt.length() +
- mInputQueueIterator->Physical().length()];
-
- MakeHtkFileName(tmp_name, mInputQueueIterator->Physical().c_str(),
- mCurrentIndexFileDir.c_str(), mCurrentIndexFileExt.c_str());
-
- mCurrentIndexFileName = tmp_name;
- }
- else
- mCurrentIndexFileName = "";
-
- //get the 3-letter suffix
- int pos_last_three_chars = mInputQueueIterator->Physical().size() - 3;
- if (pos_last_three_chars < 0) pos_last_three_chars = 0;
- //read the gzipped ascii features
- if (mInputQueueIterator->Physical().substr(pos_last_three_chars) == ".gz") {
- return ReadGzipAsciiFeatures(*mInputQueueIterator, rMatrix);
- }
-
- // read the matrix and return the result
- return ReadHTKFeatures(*mInputQueueIterator, rMatrix);
- } // ReadFullMatrix(Matrix<BaseFloat>& rMatrix)
-
-
-
- //***************************************************************************
- //***************************************************************************
- bool
- FeatureRepository::
- WriteFeatureMatrix(const Matrix<BaseFloat>& rMatrix, const std::string& filename, int targetKind, int samplePeriod)
- {
- FILE* fp = fopen(filename.c_str(),"w");
- if(NULL == fp) { Error(std::string("Cannot create file:") + filename); return false; }
-
- WriteHTKFeatures(fp, samplePeriod, targetKind, mSwapFeatures, const_cast<Matrix<BaseFloat>&>(rMatrix));
-
- fclose(fp);
-
- return true;
- }
-
-
- //***************************************************************************
- //***************************************************************************
- // private:
- int
- FeatureRepository::
- ReadHTKHeader()
- {
- // TODO
- // Change this... We should read from StkStream
- FILE* fp = mStream.fp();
-
- if (!fread(&mHeader.mNSamples, sizeof(INT_32), 1, fp)) return -1;
- if (!fread(&mHeader.mSamplePeriod, sizeof(INT_32), 1, fp)) return -1;
- if (!fread(&mHeader.mSampleSize, sizeof(INT_16), 1, fp)) return -1;
- if (!fread(&mHeader.mSampleKind, sizeof(UINT_16), 1, fp)) return -1;
-
- if (mSwapFeatures)
- {
- swap4(mHeader.mNSamples);
- swap4(mHeader.mSamplePeriod);
- swap2(mHeader.mSampleSize);
- swap2(mHeader.mSampleKind);
- }
-
- if (mHeader.mSamplePeriod < 0
- || mHeader.mSamplePeriod > 1000000
- || mHeader.mNSamples < 0
- || mHeader.mSampleSize < 0)
- {
- return -1;
- }
-
- return 0;
- }
-
-
- //***************************************************************************
- //***************************************************************************
- // private:
- int
- FeatureRepository::
- ReadHTKFeature(
- BaseFloat* pIn,
- size_t feaLen,
- bool decompress,
- BaseFloat* pScale,
- BaseFloat* pBias)
- {
- FILE* fp = mStream.fp();
-
- size_t i;
-
- if (decompress)
- {
- INT_16 s;
- // BaseFloat pScale = (xmax - xmin) / (2*32767);
- // BaseFloat pBias = (xmax + xmin) / 2;
-
- for (i = 0; i < feaLen; i++)
- {
- if (fread(&s, sizeof(INT_16), 1, fp) != 1)
- return -1;
-
- if (mSwapFeatures) swap2(s);
- pIn[i] = ((BaseFloat)s + pBias[i]) / pScale[i];
- }
-
- return 0;
- }
-
-#if !DOUBLEPRECISION
- if (fread(pIn, sizeof(FLOAT_32), feaLen, fp) != feaLen)
- return -1;
-
- if (mSwapFeatures)
- for (i = 0; i < feaLen; i++)
- swap4(pIn[i]);
-#else
- float f;
-
- for (i = 0; i < feaLen; i++)
- {
- if (fread(&f, sizeof(FLOAT_32), 1, fp) != 1)
- return -1;
-
- if (mSwapFeatures)
- swap4(f);
-
- pIn[i] = f;
- }
-#endif
- return 0;
- } // int ReadHTKFeature
-
-
-
- //***************************************************************************
- //***************************************************************************
-/* bool
- FeatureRepository::
- ReadHTKFeatures(const std::string& rFileName, Matrix<BaseFloat>& rFeatureMatrix)
- {
- std::string file_name(rFileName);
- std::string cmn_file_name;
- std::string cvn_file_name;
-
- int ext_left = mStartFrameExt;
- int ext_right = mEndFrameExt;
- int from_frame;
- int to_frame;
- int tot_frames;
- int trg_vec_size;
- int src_vec_size;
- int src_deriv_order;
- int lo_src_tgz_deriv_order;
- int i;
- int j;
- int k;
- int e;
- int coefs;
- int trg_E;
- int trg_0;
- int trg_N;
- int src_E;
- int src_0;
- int src_N;
- int comp;
- int coef_size;
- char* chptr;
-
-
-
- // read frame range definition if any ( physical_file.fea[s,e] )
- if ((chptr = strrchr(file_name.c_str(), '[')) == NULL ||
- ((i=0), sscanf(chptr, "[%d,%d]%n", &from_frame, &to_frame, &i),
- chptr[i] != '\0'))
- {
- chptr = NULL;
- }
-
- if (chptr != NULL)
- *chptr = '\0';
-
- // Experimental changes...
- // if ((strcmp(file_name.c_str(), "-"))
- // && (mpLastFileName != NULL)
- // && (!strcmp(mpLastFileName, file_name.c_str())))
- // {
- // mHeader = mLastHeader;
- // }
- // else
- // {
- // if (mpLastFileName)
- // {
- // //if (mpFp != stdin)
- // // fclose(mpFp);
- // mStream.close();
- //
- // free(mpLastFileName);
- // mpLastFileName = NULL;
- // }
-
- if ((file_name != "-" )
- && (!mLastFileName.empty())
- && (mLastFileName == file_name))
- {
- mHeader = mLastHeader;
- }
- else
- {
- if (!mLastFileName.empty())
- {
- mStream.close();
- mLastFileName = "";
- }
-
-
- // open the feature file
- mStream.open(file_name.c_str(), ios::binary);
- if (!mStream.good())
- {
- Error("Cannot open feature file: '%s'", file_name.c_str());
- }
-
-
- if (ReadHTKHeader())
- Error("Invalid HTK header in feature file: '%s'", file_name.c_str());
-
- if (mHeader.mSampleKind & PARAMKIND_C)
- {
- // File is in compressed form, scale and pBias vectors
- // are appended after HTK header.
-
- int coefs = mHeader.mSampleSize/sizeof(INT_16);
- mpA = (BaseFloat*) realloc(mpA, coefs * sizeof(BaseFloat));
- mpB = (BaseFloat*) realloc(mpB, coefs * sizeof(BaseFloat));
- if (mpA == NULL || mpB == NULL) Error("Insufficient memory");
-
- e = ReadHTKFeature(mpA, coefs, 0, 0, 0);
- e |= ReadHTKFeature(mpB, coefs, 0, 0, 0);
-
- if (e)
- Error("Cannot read feature file: '%s'", file_name.c_str());
-
- mHeader.mNSamples -= 2 * sizeof(FLOAT_32) / sizeof(INT_16);
- }
-
- // remember current settings
- mLastFileName = file_name;
- mLastHeader = mHeader;
- }
-
- if (chptr != NULL)
- *chptr = '[';
-
- if (chptr == NULL)
- { // Range [s,e] was not specified
- from_frame = 0;
- to_frame = mHeader.mNSamples-1;
- }
-
- src_deriv_order = PARAMKIND_T & mHeader.mSampleKind ? 3 :
- PARAMKIND_A & mHeader.mSampleKind ? 2 :
- PARAMKIND_D & mHeader.mSampleKind ? 1 : 0;
- src_E = (PARAMKIND_E & mHeader.mSampleKind) != 0;
- src_0 = (PARAMKIND_0 & mHeader.mSampleKind) != 0;
- src_N = ((PARAMKIND_N & mHeader.mSampleKind) != 0) * (src_E + src_0);
- comp = PARAMKIND_C & mHeader.mSampleKind;
-
- mHeader.mSampleKind &= ~PARAMKIND_C;
-
- if (mTargetKind == PARAMKIND_ANON)
- {
- mTargetKind = mHeader.mSampleKind;
- }
- else if ((mTargetKind & 077) == PARAMKIND_ANON)
- {
- mTargetKind &= ~077;
- mTargetKind |= mHeader.mSampleKind & 077;
- }
-
- trg_E = (PARAMKIND_E & mTargetKind) != 0;
- trg_0 = (PARAMKIND_0 & mTargetKind) != 0;
- trg_N =((PARAMKIND_N & mTargetKind) != 0) * (trg_E + trg_0);
-
- coef_size = comp ? sizeof(INT_16) : sizeof(FLOAT_32);
- coefs = (mHeader.mSampleSize/coef_size + src_N) /
- (src_deriv_order+1) - src_E - src_0;
- src_vec_size = (coefs + src_E + src_0) * (src_deriv_order+1) - src_N;
-
- //Is coefs dividable by 1 + number of derivatives specified in header
- if (src_vec_size * coef_size != mHeader.mSampleSize)
- {
- Error("Invalid HTK header in feature file: '%s'. "
- "mSampleSize do not match with parmKind", file_name.c_str());
- }
-
- if (mDerivOrder < 0)
- mDerivOrder = src_deriv_order;
-
-
- if ((!src_E && trg_E) || (!src_0 && trg_0) || (src_N && !trg_N) ||
- (trg_N && !trg_E && !trg_0) || (trg_N && !mDerivOrder) ||
- (src_N && !src_deriv_order && mDerivOrder) ||
- ((mHeader.mSampleKind & 077) != (mTargetKind & 077) &&
- (mHeader.mSampleKind & 077) != PARAMKIND_ANON))
- {
- char srcParmKind[64];
- char trgParmKind[64];
-
- ParmKind2Str(mHeader.mSampleKind, srcParmKind);
- ParmKind2Str(mTargetKind, trgParmKind);
- Error("Cannot convert %s to %s", srcParmKind, trgParmKind);
- }
-
- lo_src_tgz_deriv_order = LOWER_OF(src_deriv_order, mDerivOrder);
- trg_vec_size = (coefs + trg_E + trg_0) * (mDerivOrder+1) - trg_N;
-
- i = LOWER_OF(from_frame, mStartFrameExt);
- from_frame -= i;
- ext_left -= i;
-
- i = LOWER_OF(mHeader.mNSamples-to_frame-1, mEndFrameExt);
- to_frame += i;
- ext_right -= i;
-
- if (from_frame > to_frame || from_frame >= mHeader.mNSamples || to_frame< 0)
- Error("Invalid frame range for feature file: '%s'", file_name.c_str());
-
- tot_frames = to_frame - from_frame + 1 + ext_left + ext_right;
-
- // initialize matrix
- rFeatureMatrix.Init(tot_frames, trg_vec_size);
-
- // fill the matrix with features
- for (i = 0; i <= to_frame - from_frame; i++)
- {
- BaseFloat* A = mpA;
- BaseFloat* B = mpB;
- BaseFloat* mxPtr = rFeatureMatrix[i+ext_left];
-
- // seek to the desired position
- fseek(mStream.fp(),
- sizeof(HtkHeader) + (comp ? src_vec_size * 2 * sizeof(FLOAT_32) : 0)
- + (from_frame + i) * src_vec_size * coef_size,
- SEEK_SET);
-
- e = ReadHTKFeature(mxPtr, coefs, comp, A, B);
-
- mxPtr += coefs;
- A += coefs;
- B += coefs;
-
- if (src_0 && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);
- if (trg_0 && !trg_N) mxPtr++;
- if (src_E && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);
- if (trg_E && !trg_N) mxPtr++;
-
- for (j = 0; j < lo_src_tgz_deriv_order; j++)
- {
- e |= ReadHTKFeature(mxPtr, coefs, comp, A, B);
- mxPtr += coefs;
- A += coefs;
- B += coefs;
-
- if (src_0) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);
- if (trg_0) mxPtr++;
- if (src_E) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);
- if (trg_E) mxPtr++;
- }
-
- if (e)
- Error("Cannot read feature file: '%s' frame %d/%d", file_name.c_str(),
- i, to_frame - from_frame + 1);
- }
-
- // From now, coefs includes also trg_0 + trg_E !
- coefs += trg_0 + trg_E;
-
- // If extension of the matrix to the left or to the right is required,
- // perform it here
- for (i = 0; i < ext_left; i++)
- {
- memcpy(rFeatureMatrix[i],
- rFeatureMatrix[ext_left],
- (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat));
- }
-
- for (i = tot_frames - ext_right; i < tot_frames; i++)
- {
- memcpy(rFeatureMatrix[i],
- rFeatureMatrix[tot_frames - ext_right - 1],
- (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat));
- }
-
- // Sentence cepstral mean normalization
- if( (mpCmnPath == NULL)
- && !(PARAMKIND_Z & mHeader.mSampleKind)
- && (PARAMKIND_Z & mTargetKind))
- {
- // for each coefficient
- for(j=0; j < coefs; j++)
- {
- BaseFloat norm = 0.0;
- for(i=0; i < tot_frames; i++) // for each frame
- {
- norm += rFeatureMatrix[i][j - trg_N];
- //norm += fea_mx[i*trg_vec_size - trg_N + j];
- }
-
- norm /= tot_frames;
-
- for(i=0; i < tot_frames; i++) // for each frame
- rFeatureMatrix[i][j - trg_N] -= norm;
- //fea_mx[i*trg_vec_size - trg_N + j] -= norm;
- }
- }
-
- // Compute missing derivatives
- for (; src_deriv_order < mDerivOrder; src_deriv_order++)
- {
- int winLen = mDerivWinLengths[src_deriv_order];
- BaseFloat norm = 0.0;
-
- for (k = 1; k <= winLen; k++)
- {
- norm += 2 * k * k;
- }
-
- // for each frame
- for (i=0; i < tot_frames; i++)
- {
- // for each coefficient
- for (j=0; j < coefs; j++)
- {
- //BaseFloat* src = fea_mx + i*trg_vec_size + src_deriv_order*coefs - trg_N + j;
- BaseFloat* src = &rFeatureMatrix[i][src_deriv_order*coefs - trg_N + j];
-
- *(src + coefs) = 0.0;
-
- if (i < winLen || i >= tot_frames-winLen)
- { // boundaries need special treatment
- for (k = 1; k <= winLen; k++)
- {
- *(src+coefs) += k*(src[ LOWER_OF(tot_frames-1-i,k)*rFeatureMatrix.Stride()]
- -src[-LOWER_OF(i, k)*rFeatureMatrix.Stride()]);
- }
- }
- else
- { // otherwise use more efficient code
- for (k = 1; k <= winLen; k++)
- {
- *(src+coefs) += k*(src[ k * rFeatureMatrix.Stride()]
- -src[-k * rFeatureMatrix.Stride()]);
- }
- }
- *(src + coefs) /= norm;
- }
- }
- }
-
- mHeader.mNSamples = tot_frames;
- mHeader.mSampleSize = trg_vec_size * sizeof(FLOAT_32);
- mHeader.mSampleKind = mTargetKind & ~(PARAMKIND_D | PARAMKIND_A | PARAMKIND_T);
-
-
- ////////////////////////////////////////////////////////////////////////////
- /////////////// Cepstral mean and variance normalization ///////////////////
- ////////////////////////////////////////////////////////////////////////////
- //.........................................................................
- if (mpCmnPath != NULL
- && mpCmnMask != NULL)
- {
- // retrieve file name
- ProcessMask(file_name, mpCmnMask, cmn_file_name);
- // add the path correctly
- cmn_file_name.insert(0, "/");
- cmn_file_name.insert(0, mpCmnPath);
-
- // read the file
- ReadCepsNormFile(cmn_file_name.c_str(), &mpLastCmnFile, &mpCmn,
- mHeader.mSampleKind & ~PARAMKIND_Z, CNF_Mean, coefs);
-
- // recompute feature values
- for (i=0; i < tot_frames; i++)
- {
- for (j=trg_N; j < coefs; j++)
- {
- rFeatureMatrix[i][j - trg_N] -= mpCmn[j];
- }
- }
- }
-
- mHeader.mSampleKind |= mDerivOrder==3 ? PARAMKIND_D | PARAMKIND_A | PARAMKIND_T :
- mDerivOrder==2 ? PARAMKIND_D | PARAMKIND_A :
- mDerivOrder==1 ? PARAMKIND_D : 0;
-
- //.........................................................................
- if (mpCvnPath != NULL
- && mpCvnMask != NULL)
- {
- // retrieve file name
- ProcessMask(file_name, mpCvnMask, cvn_file_name);
- // add the path correctly
- cvn_file_name.insert(0, "/");
- cvn_file_name.insert(0, mpCvnPath);
-
- // read the file
- ReadCepsNormFile(cvn_file_name.c_str(), &mpLastCvnFile, &mpCvn,
- mHeader.mSampleKind, CNF_Variance, trg_vec_size);
-
- // recompute feature values
- for (i=0; i < tot_frames; i++)
- {
- for (j=trg_N; j < trg_vec_size; j++)
- {
- rFeatureMatrix[i][j - trg_N] *= mpCvn[j];
- }
- }
- }
-
- //.........................................................................
- // process the global covariance file
- if (mpCvgFile != NULL)
- {
- ReadCepsNormFile(mpCvgFile, &mpLastCvgFile, &mpCvg,
- -1, CNF_VarScale, trg_vec_size);
-
- // recompute feature values
- for (i=0; i < tot_frames; i++)
- {
- for (j=trg_N; j < trg_vec_size; j++)
- {
- rFeatureMatrix[i][j - trg_N] *= mpCvg[j];
- }
- }
- }
-
- return true;
- }
-*/
-
- //***************************************************************************
- //***************************************************************************
-
-
-
-
-
- //***************************************************************************
- //***************************************************************************
- bool
- FeatureRepository::
- ReadHTKFeatures(const FileListElem& rFileNameRecord,
- Matrix<BaseFloat>& rFeatureMatrix)
- {
- std::string file_name(rFileNameRecord.Physical());
- std::string cmn_file_name;
- std::string cvn_file_name;
-
- int ext_left = mStartFrameExt;
- int ext_right = mEndFrameExt;
- int from_frame;
- int to_frame;
- int tot_frames;
- int trg_vec_size;
- int src_vec_size;
- int src_deriv_order;
- int lo_src_tgz_deriv_order;
- int i;
- int j;
- int k;
- int e;
- int coefs;
- int trg_E;
- int trg_0;
- int trg_N;
- int src_E;
- int src_0;
- int src_N;
- int comp;
- int coef_size;
- char* chptr;
-
-
- TIMER_START(mTim);
-
- // read frame range definition if any ( physical_file.fea[s,e] )
- if ((chptr = strrchr((char*)file_name.c_str(), '[')) == NULL ||
- ((i=0), sscanf(chptr, "[%d,%d]%n", &from_frame, &to_frame, &i),
- chptr[i] != '\0'))
- {
- chptr = NULL;
- }
-
- if (chptr != NULL)
- *chptr = '\0';
-
-
- if ((file_name != "-" )
- && (!mLastFileName.empty())
- && (mLastFileName == file_name))
- {
- mHeader = mLastHeader;
- }
- else
- {
- if (!mLastFileName.empty())
- {
- mStream.close();
- mLastFileName = "";
- }
-
-
- // open the feature file
- mStream.open(file_name.c_str(), std::ios::binary);
- if (!mStream.good())
- {
- throw std::runtime_error(std::string("Cannot open feature file: '")
- + file_name.c_str() + "'");
- }
-
-
- if (ReadHTKHeader()) {
- throw std::runtime_error(std::string("Invalid HTK header in feature file: '")
- + file_name.c_str() + "'");
- }
-
- if (mHeader.mSampleKind & PARAMKIND_C)
- {
- // File is in compressed form, scale and pBias vectors
- // are appended after HTK header.
- coefs = mHeader.mSampleSize/sizeof(INT_16);
-
- mpA = (BaseFloat*) realloc(mpA, coefs * sizeof(BaseFloat));
- mpB = (BaseFloat*) realloc(mpB, coefs * sizeof(BaseFloat));
-
- if (mpA == NULL || mpB == NULL) {
- throw std::runtime_error("Insufficient memory");
- }
-
- e = ReadHTKFeature(mpA, coefs, 0, 0, 0);
- e |= ReadHTKFeature(mpB, coefs, 0, 0, 0);
-
- if (e) {
- throw std::runtime_error(std::string("Cannot read feature file: '")
- + file_name.c_str() + "'");
- }
-
- mHeader.mNSamples -= 2 * sizeof(FLOAT_32) / sizeof(INT_16);
- }
-
- // remember current settings
- mLastFileName = file_name;
- mLastHeader = mHeader;
- }
-
- if (chptr != NULL) {
- *chptr = '[';
- }
-
- if (chptr == NULL) {
- // Range [s,e] was not specified
- from_frame = 0;
- to_frame = mHeader.mNSamples-1;
- }
-
- src_deriv_order = PARAMKIND_T & mHeader.mSampleKind ? 3 :
- PARAMKIND_A & mHeader.mSampleKind ? 2 :
- PARAMKIND_D & mHeader.mSampleKind ? 1 : 0;
- src_E = (PARAMKIND_E & mHeader.mSampleKind) != 0;
- src_0 = (PARAMKIND_0 & mHeader.mSampleKind) != 0;
- src_N = ((PARAMKIND_N & mHeader.mSampleKind) != 0) * (src_E + src_0);
- comp = PARAMKIND_C & mHeader.mSampleKind;
-
- mHeader.mSampleKind &= ~PARAMKIND_C;
-
- if (mTargetKind == PARAMKIND_ANON)
- {
- mTargetKind = mHeader.mSampleKind;
- }
- else if ((mTargetKind & 077) == PARAMKIND_ANON)
- {
- mTargetKind &= ~077;
- mTargetKind |= mHeader.mSampleKind & 077;
- }
-
- trg_E = (PARAMKIND_E & mTargetKind) != 0;
- trg_0 = (PARAMKIND_0 & mTargetKind) != 0;
- trg_N =((PARAMKIND_N & mTargetKind) != 0) * (trg_E + trg_0);
-
- coef_size = comp ? sizeof(INT_16) : sizeof(FLOAT_32);
- coefs = (mHeader.mSampleSize/coef_size + src_N) /
- (src_deriv_order+1) - src_E - src_0;
- src_vec_size = (coefs + src_E + src_0) * (src_deriv_order+1) - src_N;
-
- //Is coefs dividable by 1 + number of derivatives specified in header
- if (src_vec_size * coef_size != mHeader.mSampleSize)
- {
- throw std::runtime_error(std::string("Invalid HTK header in feature file: '")
- + file_name + "' mSampleSize do not match with parmKind");
- }
-
- if (mDerivOrder < 0)
- mDerivOrder = src_deriv_order;
-
-
- if ((!src_E && trg_E) || (!src_0 && trg_0) || (src_N && !trg_N) ||
- (trg_N && !trg_E && !trg_0) || (trg_N && !mDerivOrder) ||
- (src_N && !src_deriv_order && mDerivOrder) ||
- ((mHeader.mSampleKind & 077) != (mTargetKind & 077) &&
- (mHeader.mSampleKind & 077) != PARAMKIND_ANON))
- {
- char srcParmKind[64];
- char trgParmKind[64];
- memset(srcParmKind,0,64);
- memset(trgParmKind,0,64);
-
- ParmKind2Str(mHeader.mSampleKind, srcParmKind);
- ParmKind2Str(mTargetKind, trgParmKind);
- throw std::runtime_error(std::string("Cannot convert ") + srcParmKind
- + " to " + trgParmKind);
- }
-
- lo_src_tgz_deriv_order = std::min(src_deriv_order, mDerivOrder);
- trg_vec_size = (coefs + trg_E + trg_0) * (mDerivOrder+1) - trg_N;
-
- i = std::min(from_frame, mStartFrameExt);
- from_frame -= i;
- ext_left -= i;
-
- i = std::min(mHeader.mNSamples-to_frame-1, mEndFrameExt);
- to_frame += i;
- ext_right -= i;
-
- if (from_frame > to_frame || from_frame >= mHeader.mNSamples || to_frame< 0)
- throw std::runtime_error(std::string("Invalid frame range for feature file: '")
- + file_name.c_str() + "'");
-
- tot_frames = to_frame - from_frame + 1 + ext_left + ext_right;
-
-
- TIMER_END(mTim,mTimeOpen);
-
-
- // initialize matrix
- rFeatureMatrix.Init(tot_frames, trg_vec_size, false);
-
- // fill the matrix with features
- for (i = 0; i <= to_frame - from_frame; i++)
- {
- BaseFloat* A = mpA;
- BaseFloat* B = mpB;
- BaseFloat* mxPtr = rFeatureMatrix.pRowData(i+ext_left);
-
- TIMER_START(mTim);
- // seek to the desired position
- fseek(mStream.fp(),
- sizeof(HtkHeader) + (comp ? src_vec_size * 2 * sizeof(FLOAT_32) : 0)
- + (from_frame + i) * src_vec_size * coef_size,
- SEEK_SET);
- TIMER_END(mTim,mTimeSeek);
-
- TIMER_START(mTim);
- // read
- e = ReadHTKFeature(mxPtr, coefs, comp, A, B);
- TIMER_END(mTim,mTimeRead);
-
- mxPtr += coefs;
- A += coefs;
- B += coefs;
-
- if (src_0 && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);
- if (trg_0 && !trg_N) mxPtr++;
- if (src_E && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);
- if (trg_E && !trg_N) mxPtr++;
-
- for (j = 0; j < lo_src_tgz_deriv_order; j++)
- {
- e |= ReadHTKFeature(mxPtr, coefs, comp, A, B);
- mxPtr += coefs;
- A += coefs;
- B += coefs;
-
- if (src_0) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);
- if (trg_0) mxPtr++;
- if (src_E) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++);
- if (trg_E) mxPtr++;
- }
-
- if (e) {
- std::cout << mHeader.mNSamples << "\n";
- std::cout << 2 * sizeof(FLOAT_32) / sizeof(INT_16) << "\n";
- std::cout << "from" << from_frame << "to" << to_frame << "i" << i << "\n";
-
- std::ostringstream s;
- s << i << "/" << to_frame - from_frame + 1, s.str();
- throw std::runtime_error(std::string("Cannot read feature file: '")
- + file_name + "' frame " + s.str());
- }
- }
-
- // From now, coefs includes also trg_0 + trg_E !
- coefs += trg_0 + trg_E;
-
- // If extension of the matrix to the left or to the right is required,
- // perform it here
- for (i = 0; i < ext_left; i++)
- {
- memcpy(rFeatureMatrix.pRowData(i),
- rFeatureMatrix.pRowData(ext_left),
- (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat));
- }
-
- for (i = tot_frames - ext_right; i < tot_frames; i++)
- {
- memcpy(rFeatureMatrix.pRowData(i),
- rFeatureMatrix.pRowData(tot_frames - ext_right - 1),
- (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat));
- }
-
- // Sentence cepstral mean normalization
- if( (mpCmnPath == NULL)
- && !(PARAMKIND_Z & mHeader.mSampleKind)
- && (PARAMKIND_Z & mTargetKind))
- {
- // for each coefficient
- for(j=0; j < coefs; j++)
- {
- BaseFloat norm = 0.0;
- for(i=0; i < tot_frames; i++) // for each frame
- {
- norm += rFeatureMatrix[i][j - trg_N];
- //norm += fea_mx[i*trg_vec_size - trg_N + j];
- }
-
- norm /= tot_frames;
-
- for(i=0; i < tot_frames; i++) // for each frame
- rFeatureMatrix[i][j - trg_N] -= norm;
- //fea_mx[i*trg_vec_size - trg_N + j] -= norm;
- }
- }
-
- // Compute missing derivatives
- for (; src_deriv_order < mDerivOrder; src_deriv_order++)
- {
- int winLen = mDerivWinLengths[src_deriv_order];
- BaseFloat norm = 0.0;
-
- for (k = 1; k <= winLen; k++)
- {
- norm += 2 * k * k;
- }
-
- // for each frame
- for (i=0; i < tot_frames; i++)
- {
- // for each coefficient
- for (j=0; j < coefs; j++)
- {
- //BaseFloat* src = fea_mx + i*trg_vec_size + src_deriv_order*coefs - trg_N + j;
- BaseFloat* src = &rFeatureMatrix[i][src_deriv_order*coefs - trg_N + j];
-
- *(src + coefs) = 0.0;
-
- if (i < winLen || i >= tot_frames-winLen)
- { // boundaries need special treatment
- for (k = 1; k <= winLen; k++)
- {
- *(src+coefs) += k*(src[ std::min(tot_frames-1-i,k)*rFeatureMatrix.Stride()]
- -src[-std::min(i, k)*rFeatureMatrix.Stride()]);
- }
- }
- else
- { // otherwise use more efficient code
- for (k = 1; k <= winLen; k++)
- {
- *(src+coefs) += k*(src[ k * rFeatureMatrix.Stride()]
- -src[-k * rFeatureMatrix.Stride()]);
- }
- }
- *(src + coefs) /= norm;
- }
- }
- }
-
- mHeader.mNSamples = tot_frames;
- mHeader.mSampleSize = trg_vec_size * sizeof(FLOAT_32);
- mHeader.mSampleKind = mTargetKind & ~(PARAMKIND_D | PARAMKIND_A | PARAMKIND_T);
-
-
- TIMER_START(mTim);
- ////////////////////////////////////////////////////////////////////////////
- /////////////// Cepstral mean and variance normalization ///////////////////
- ////////////////////////////////////////////////////////////////////////////
- //.........................................................................
- if (mpCmnPath != NULL
- && mpCmnMask != NULL)
- {
- // retrieve file name
- ProcessMask(rFileNameRecord.Logical(), mpCmnMask, cmn_file_name);
- // add the path correctly
-
- if(cmn_file_name == "") {
- throw std::runtime_error("CMN Matching failed");
- }
-
- cmn_file_name.insert(0, "/");
- cmn_file_name.insert(0, mpCmnPath);
-
- // read the file
- ReadCepsNormFile(cmn_file_name.c_str(), &mpLastCmnFile, &mpCmn,
- mHeader.mSampleKind & ~PARAMKIND_Z, CNF_Mean, coefs);
-
- // recompute feature values
- for (i=0; i < tot_frames; i++)
- {
- for (j=trg_N; j < coefs; j++)
- {
- rFeatureMatrix[i][j - trg_N] -= mpCmn[j];
- }
- }
- }
-
- mHeader.mSampleKind |= mDerivOrder==3 ? PARAMKIND_D | PARAMKIND_A | PARAMKIND_T :
- mDerivOrder==2 ? PARAMKIND_D | PARAMKIND_A :
- mDerivOrder==1 ? PARAMKIND_D : 0;
-
- //.........................................................................
- if (mpCvnPath != NULL
- && mpCvnMask != NULL)
- {
- // retrieve file name
- ProcessMask(rFileNameRecord.Logical(), mpCvnMask, cvn_file_name);
- // add the path correctly
- cvn_file_name.insert(0, "/");
- cvn_file_name.insert(0, mpCvnPath);
-
- // read the file
- ReadCepsNormFile(cvn_file_name.c_str(), &mpLastCvnFile, &mpCvn,
- mHeader.mSampleKind, CNF_Variance, trg_vec_size);
-
- // recompute feature values
- for (i=0; i < tot_frames; i++)
- {
- for (j=trg_N; j < trg_vec_size; j++)
- {
- rFeatureMatrix[i][j - trg_N] *= mpCvn[j];
- }
- }
- }
-
- //.........................................................................
- // process the global covariance file
- if (mpCvgFile != NULL)
- {
- ReadCepsNormFile(mpCvgFile, &mpLastCvgFile, &mpCvg,
- -1, CNF_VarScale, trg_vec_size);
-
- // recompute feature values
- for (i=0; i < tot_frames; i++)
- {
- for (j=trg_N; j < trg_vec_size; j++)
- {
- rFeatureMatrix[i][j - trg_N] *= mpCvg[j];
- }
- }
- }
-
- TIMER_END(mTim,mTimeNormalize);
-
- return true;
- }
-
-
- //***************************************************************************
- //***************************************************************************
- int
- FeatureRepository::
- ReadParmKind(const char *str, bool checkBrackets)
- {
- unsigned int i;
- int parmKind =0;
- int slen = strlen(str);
-
- if (checkBrackets)
- {
- if (str[0] != '<' || str[slen-1] != '>') return -1;
- str++; slen -= 2;
- }
-
- for (; slen >= 0 && str[slen-2] == '_'; slen -= 2)
- {
- parmKind |= str[slen-1] == 'E' ? PARAMKIND_E :
- str[slen-1] == 'N' ? PARAMKIND_N :
- str[slen-1] == 'D' ? PARAMKIND_D :
- str[slen-1] == 'A' ? PARAMKIND_A :
- str[slen-1] == 'C' ? PARAMKIND_C :
- str[slen-1] == 'Z' ? PARAMKIND_Z :
- str[slen-1] == 'K' ? PARAMKIND_K :
- str[slen-1] == '0' ? PARAMKIND_0 :
- str[slen-1] == 'V' ? PARAMKIND_V :
- str[slen-1] == 'T' ? PARAMKIND_T : -1;
-
- if (parmKind == -1) return -1;
- }
-
- for (i = 0; i < sizeof(mpParmKindNames) / sizeof(char*); i++)
- {
- if (!strncmp(str, mpParmKindNames[i], slen))
- return parmKind | i;
- }
- return -1;
- }
-
-
-
-
- //***************************************************************************
- //***************************************************************************
- int
- FeatureRepository::
- WriteHTKHeader (FILE * pOutFp, HtkHeader header, bool swap)
- {
- int cc;
-
- if (swap) {
- swap4(header.mNSamples);
- swap4(header.mSamplePeriod);
- swap2(header.mSampleSize);
- swap2(header.mSampleKind);
- }
-
- fseek (pOutFp, 0L, SEEK_SET);
- cc = fwrite(&header, sizeof(HtkHeader), 1, pOutFp);
-
- if (swap) {
- swap4(header.mNSamples);
- swap4(header.mSamplePeriod);
- swap2(header.mSampleSize);
- swap2(header.mSampleKind);
- }
-
- return cc == 1 ? 0 : -1;
- }
-
-
- //***************************************************************************
- //***************************************************************************
- int
- FeatureRepository::
- WriteHTKFeature(
- FILE * pOutFp,
- FLOAT * pOut,
- size_t feaLen,
- bool swap,
- bool compress,
- FLOAT* pScale,
- FLOAT* pBias)
- {
- size_t i;
- size_t cc = 0;
-
-
- if (compress)
- {
- INT_16 s;
-
- for (i = 0; i < feaLen; i++)
- {
- s = pOut[i] * pScale[i] - pBias[i];
- if (swap)
- swap2(s);
- cc += fwrite(&s, sizeof(INT_16), 1, pOutFp);
- }
-
- } else {
- #if !DOUBLEPRECISION
- if (swap)
- for (i = 0; i < feaLen; i++)
- swap4(pOut[i]);
-
- cc = fwrite(pOut, sizeof(FLOAT_32), feaLen, pOutFp);
-
- if (swap)
- for (i = 0; i < feaLen; i++)
- swap4(pOut[i]);
- #else
- FLOAT_32 f;
-
- for (i = 0; i < feaLen; i++)
- {
- f = pOut[i];
- if (swap)
- swap4(f);
- cc += fwrite(&f, sizeof(FLOAT_32), 1, pOutFp);
- }
- #endif
- }
- return cc == feaLen ? 0 : -1;
- }
-
- //***************************************************************************
- //***************************************************************************
- int
- FeatureRepository::
- WriteHTKFeatures(
- FILE * pOutFp,
- FLOAT * pOut,
- int nCoeffs,
- int nSamples,
- int samplePeriod,
- int targetKind,
- bool swap)
- {
- HtkHeader header;
- int i, j;
- FLOAT *pScale = NULL;
- FLOAT *pBias = NULL;
-
- header.mNSamples = nSamples + ((targetKind & PARAMKIND_C) ? 2 * sizeof(FLOAT_32) / sizeof(INT_16) : 0);
- header.mSamplePeriod = samplePeriod;
- header.mSampleSize = nCoeffs * ((targetKind & PARAMKIND_C) ? sizeof(INT_16) : sizeof(FLOAT_32));;
- header.mSampleKind = targetKind;
-
- WriteHTKHeader (pOutFp, header, swap);
-
- if(targetKind & PARAMKIND_C) {
- pScale = (FLOAT*) malloc(nCoeffs * sizeof(FLOAT));
- pBias = (FLOAT*) malloc(nCoeffs * sizeof(FLOAT));
- if (pScale == NULL || pBias == NULL) Error("Insufficient memory");
-
- for(i = 0; i < nCoeffs; i++) {
- float xmin, xmax;
- xmin = xmax = pOut[i];
- for(j = 1; j < nSamples; j++) {
- if(pOut[j*nCoeffs+i] > xmax) xmax = pOut[j*nCoeffs+i];
- if(pOut[j*nCoeffs+i] < xmin) xmin = pOut[j*nCoeffs+i];
- }
- pScale[i] = (2*32767) / (xmax - xmin);
- pBias[i] = pScale[i] * (xmax + xmin) / 2;
-
-
- }
- if (WriteHTKFeature(pOutFp, pScale, nCoeffs, swap, false, 0, 0)
- || WriteHTKFeature(pOutFp, pBias, nCoeffs, swap, false, 0, 0)) {
- return -1;
- }
- }
- for(j = 0; j < nSamples; j++) {
- if (WriteHTKFeature(pOutFp, &pOut[j*nCoeffs], nCoeffs, swap, targetKind & PARAMKIND_C, pScale, pBias)) {
- return -1;
- }
- }
- return 0;
- }
-
-
- //***************************************************************************
- //***************************************************************************
- int
- FeatureRepository::
- WriteHTKFeatures(
- FILE * pOutFp,
- int samplePeriod,
- int targetKind,
- bool swap,
- Matrix<BaseFloat>& rFeatureMatrix)
- {
- HtkHeader header;
- size_t i, j;
- FLOAT *p_scale = NULL;
- FLOAT *p_bias = NULL;
- size_t n_samples = rFeatureMatrix.Rows();
- size_t n_coeffs = rFeatureMatrix.Cols();
-
- header.mNSamples = n_samples + ((targetKind & PARAMKIND_C) ? 2 * sizeof(FLOAT_32) / sizeof(INT_16) : 0);
- header.mSamplePeriod = samplePeriod;
- header.mSampleSize = n_coeffs * ((targetKind & PARAMKIND_C) ? sizeof(INT_16) : sizeof(FLOAT_32));;
- header.mSampleKind = targetKind;
-
- WriteHTKHeader (pOutFp, header, swap);
-
- if(targetKind & PARAMKIND_C) {
- p_scale = (FLOAT*) malloc(n_coeffs * sizeof(FLOAT));
- p_bias = (FLOAT*) malloc(n_coeffs * sizeof(FLOAT));
- if (p_scale == NULL || p_bias == NULL) Error("Insufficient memory");
-
- for(i = 0; i < n_coeffs; i++) {
- float xmin, xmax;
- xmin = xmax = rFeatureMatrix[0][i];
-
- for(j = 1; j < n_samples; j++) {
- if(rFeatureMatrix[j][i] > xmax) xmax = rFeatureMatrix[j][i];
- if(rFeatureMatrix[j][i] < xmin) xmin = rFeatureMatrix[j][i];
- }
-
- p_scale[i] = (2*32767) / (xmax - xmin);
- p_bias[i] = p_scale[i] * (xmax + xmin) / 2;
- }
-
- if (WriteHTKFeature(pOutFp, p_scale, n_coeffs, swap, false, 0, 0)
- || WriteHTKFeature(pOutFp, p_bias, n_coeffs, swap, false, 0, 0)) {
- return -1;
- }
- }
-
- for(j = 0; j < n_samples; j++) {
- if (WriteHTKFeature(pOutFp, rFeatureMatrix[j].pData(), n_coeffs, swap, targetKind & PARAMKIND_C, p_scale, p_bias)) {
- return -1;
- }
- }
-
- return 0;
- }
-
- //***************************************************************************
- //***************************************************************************
-
-
- bool
- FeatureRepository::
- ReadGzipAsciiFeatures(const FileListElem& rFileNameRecord, Matrix<BaseFloat>& rFeatureMatrix)
- {
- //build the command
- std::string cmd("gunzip -c "); cmd += rFileNameRecord.Physical();
-
- //define buffer
- const int buf_size=262144;
- char buf[buf_size];
- char vbuf[2*buf_size];
-
- TIMER_START(mTim);
- //open the pipe
- FILE* fp = popen(cmd.c_str(),"r");
- if(fp == NULL) {
- //2nd try...
- Warning(std::string("2nd try to open pipe: ")+cmd);
- sleep(5);
- fp = popen(cmd.c_str(),"r");
- if(fp == NULL) {
- KALDI_ERR << "Cannot open pipe: " << cmd;
- }
- }
- setvbuf(fp,vbuf,_IOFBF,2*buf_size);
- TIMER_END(mTim,mTimeOpen);
-
- //string will stay allocated across calls
- static std::string line; line.resize(0);
-
- //define matrix storage
- static int cols = 131072;
- std::list<std::vector<BaseFloat> > matrix(1);
- matrix.front().reserve(cols);
-
- //read all the lines to a vector
- int line_ctr=1;
- while(1) {
- TIMER_START(mTim);
- if(NULL == fgets(buf,buf_size,fp)) break;
- TIMER_END(mTim,mTimeRead);
-
- line += buf;
- if(*(line.rbegin()) == '\n' || feof(fp)) {
- //parse the line of numbers
- TIMER_START(mTim);
- const char* ptr = line.c_str();
- char* end;
- while(1) {
- //skip whitespace
- while(isspace(*ptr)) ptr++;
- if(*ptr == 0) break;
- //check that a number follows
- switch(*ptr) {
- case '0': case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- case '.': case '+': case '-':
- break;
- default : KALDI_ERR << "A number was expected:" << ptr
- << " reading from" << cmd;
- exit(1);
- }
- //read a number
- BaseFloat val = strtof(ptr,&end); ptr=end;
- matrix.back().push_back(val);
- }
- TIMER_END(mTim,mTimeNormalize);
- //we have the line of numbers, insert empty row to matrix
- if(matrix.back().size() > 0 && !feof(fp)) {
- matrix.push_back(std::vector<BaseFloat>());
- matrix.back().reserve(matrix.front().size());
- }
- //dispose the current line
- line.resize(0);//but stay allocated...
- line_ctr++;
- }
- }
- if(matrix.back().size() == 0) matrix.pop_back();
-
- //get matrix dimensions
- int rows = matrix.size();
- /*int*/ cols = matrix.front().size();
-
- //define interators
- std::list<std::vector<BaseFloat> >::iterator it_r;
- std::vector<BaseFloat>::iterator it_c;
-
- //check that all lines have same size
- int i;
- for(i=0,it_r=matrix.begin(); it_r != matrix.end(); ++i,++it_r) {
- if(it_r->size() != cols) {
- KALDI_ERR << "All rows must have same dimension, 1st line cols: " << cols
- << ", " << i << "th line cols: " << it_r->size();
- }
- }
-
- //copy data to matrix
- TIMER_START(mTim);
- rFeatureMatrix.Init(rows,cols);
- int r,c;
- for(r=0,it_r=matrix.begin(); it_r!=matrix.end(); ++r,++it_r) {
- for(c=0,it_c=it_r->begin(); it_c!=it_r->end(); ++c,++it_c) {
- rFeatureMatrix(r,c) = *it_c;
- }
- }
- TIMER_END(mTim,mTimeSeek);
-
- //close the pipe
- if(pclose(fp) == -1) {
- KALDI_ERR << "Cannot close pipe: " << cmd;
- }
-
- return true;
- }
-
-
- //***************************************************************************
- //***************************************************************************
-
-} // namespace TNet