diff options
Diffstat (limited to 'htk_io/src/KaldiLib/Features.cc')
| -rw-r--r-- | htk_io/src/KaldiLib/Features.cc | 1798 |
1 files changed, 1798 insertions, 0 deletions
diff --git a/htk_io/src/KaldiLib/Features.cc b/htk_io/src/KaldiLib/Features.cc new file mode 100644 index 0000000..64b63e8 --- /dev/null +++ b/htk_io/src/KaldiLib/Features.cc @@ -0,0 +1,1798 @@ + +//enable feature repository profiling +#define PROFILING 1 + +#include <sstream> +#include <map> +#include <list> +#include <cstdio> + +#include "Features.h" +#include "Tokenizer.h" +#include "StkMatch.h" +#include "Types.h" + + + +namespace TNet +{ + const char + FeatureRepository:: + mpParmKindNames[13][16] = + { + {"WAVEFORM"}, + {"LPC"}, + {"LPREFC"}, + {"LPCEPSTRA"}, + {"LPDELCEP"}, + {"IREFC"}, + {"MFCC"}, + {"FBANK"}, + {"MELSPEC"}, + {"USER"}, + {"DISCRETE"}, + {"PLP"}, + {"ANON"} + }; + + //*************************************************************************** + //*************************************************************************** + + FileListElem:: + FileListElem(const std::string & rFileName) + { + std::string::size_type pos; + + mLogical = rFileName; + mWeight = 1.0; + + // some slash-backslash replacement hack + for (size_t i = 0; i < mLogical.size(); i++) { + if (mLogical[i] == '\\') { + mLogical[i] = '/'; + } + } + + // read sentence weight definition if any ( physical_file.fea[s,e]{weight} ) + if ((pos = mLogical.find('{')) != std::string::npos) + { + std::string tmp_weight(mLogical.begin() + pos + 1, mLogical.end()); + std::stringstream tmp_ss(tmp_weight); + + tmp_ss >> mWeight; + mLogical.erase(pos); + } + + // look for "=" symbol and if found, split it + if ((pos = mLogical.find('=')) != std::string::npos) + { + // copy all from mLogical[pos+1] till the end to mPhysical + mPhysical.assign(mLogical.begin() + pos + 1, mLogical.end()); + // erase all from pos + 1 till the end from mLogical + mLogical.erase(pos); + // trim the leading and trailing spaces + Trim(mPhysical); + Trim(mLogical); + } + else + { + // trim the leading and trailing spaces + Trim(mLogical); + + mPhysical = mLogical; + } + } + + + //########################################################################### + //########################################################################### + // FeatureRepository section + //########################################################################### + //########################################################################### + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + ReadCepsNormFile( + const char * pFileName, + char ** pLastFileName, + BaseFloat ** vec_buff, + int sampleKind, + CNFileType type, + int coefs) + { + FILE* fp; + int i; + char s1[64]; + char s2[64]; + const char* typeStr = (type == CNF_Mean ? "MEAN" : + type == CNF_Variance ? "VARIANCE" : "VARSCALE"); + + const char* typeStr2 = (type == CNF_Mean ? "CMN" : + type == CNF_Variance ? "CVN" : "VarScale"); + + if (*pLastFileName != NULL && !strcmp(*pLastFileName, pFileName)) { + return; + } + free(*pLastFileName); + *pLastFileName=strdup(pFileName); + *vec_buff = (BaseFloat*) realloc(*vec_buff, coefs * sizeof(BaseFloat)); + + if (*pLastFileName == NULL || *vec_buff== NULL) + throw std::runtime_error("Insufficient memory"); + + if ((fp = fopen(pFileName, "r")) == NULL) { + throw std::runtime_error(std::string("Cannot open ") + typeStr2 + + " pFileName: '" + pFileName + "'"); + } + + if ((type != CNF_VarScale + && (fscanf(fp, " <%64[^>]> <%64[^>]>", s1, s2) != 2 + || strcmp(StrToUpper(s1), "CEPSNORM") + || ReadParmKind(s2, false) != sampleKind)) + || fscanf(fp, " <%64[^>]> %d", s1, &i) != 2 + || strcmp(StrToUpper(s1), typeStr) + || i != coefs) + { + ParmKind2Str(sampleKind, s2); + + //std::cout << "[[[TADY!!!!]]]" << pFileName << "\n" << std::flush; + + throw std::runtime_error(std::string("") + + (type == CNF_VarScale ? "" : "<CEPSNORM> <") + + (type == CNF_VarScale ? "" : s2) + + (type == CNF_VarScale ? "" : ">") + + " <" + typeStr + " ... expected in " + typeStr2 + + " file " + pFileName); + } + + for (i = 0; i < coefs; i++) { + if (fscanf(fp, " "FLOAT_FMT, *vec_buff+i) != 1) { + if (fscanf(fp, "%64s", s2) == 1) { + throw std::runtime_error(std::string("Decimal number expected but '") + + s2 + "' found in " + typeStr2 + " file " + pFileName); + } + else if (feof(fp)) { + throw std::runtime_error(std::string("Unexpected end of ") + + typeStr2 + " file "+ pFileName); + } + else { + throw std::runtime_error(std::string("Cannot read ") + typeStr2 + + " file " + pFileName); + } + } + + if (type == CNF_Variance) + (*vec_buff)[i] = BaseFloat(1 / sqrt((*vec_buff)[i])); + else if (type == CNF_VarScale) + (*vec_buff)[i] = BaseFloat(sqrt((*vec_buff)[i])); + } + + if (fscanf(fp, "%64s", s2) == 1) + { + throw std::runtime_error(std::string("End of file expected but '") + + s2 + "' found in " + typeStr2 + " file " + pFileName); + } + + fclose(fp); + } // ReadCepsNormFile(...) + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + HtkFilter(const char* pFilter, const char* pValue, FeatureRepository& rOut) + { + std::list<FileListElem>::iterator it; + std::string str; + + rOut.mSwapFeatures = mSwapFeatures; + rOut.mStartFrameExt = mStartFrameExt; + rOut.mEndFrameExt = mEndFrameExt; + rOut.mTargetKind = mTargetKind; + rOut.mDerivOrder = mDerivOrder; + rOut.mDerivWinLengths = mDerivWinLengths; + + rOut.mpCvgFile = mpCvgFile; + rOut.mpCmnPath = mpCmnPath; + rOut.mpCmnMask = mpCmnMask; + rOut.mpCvnPath = mpCvnPath; + rOut.mpCvnMask = mpCvnMask; + + rOut.mInputQueue.clear(); + + // go through all records and check the mask + for (it=mInputQueue.begin(); it!= mInputQueue.end(); ++it) { + if (pFilter == NULL + || (ProcessMask(it->Logical(), pFilter, str) && (str == pValue))) { + rOut.mInputQueue.push_back(*it); + } + } + + // set the queue position to the begining + rOut.mInputQueueIterator = mInputQueue.end(); + + rOut.mCurrentIndexFileName = ""; + rOut.mCurrentIndexFileDir = ""; + rOut.mCurrentIndexFileExt = ""; + + mStream.close(); + mStream.clear(); + + rOut.mpLastFileName = NULL; + rOut.mLastFileName = ""; + rOut.mpLastCmnFile = NULL; + rOut.mpLastCvnFile = NULL; + rOut.mpLastCvgFile = NULL; + rOut.mpCmn = NULL; + rOut.mpCvn = NULL; + rOut.mpCvg = NULL; + rOut.mpA = NULL; + rOut.mpB = NULL; + + } + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + HtkSelection(const char* pFilter, std::list< std::string >& rOut) + { + std::map< std::string, bool> aux_map; + std::map< std::string, bool>::iterator map_it; + std::list<FileListElem>::iterator it; + std::string str; + + rOut.clear(); + + if(pFilter != NULL) { + // go through all records and check the mask + for (it=mInputQueue.begin(); it!= mInputQueue.end(); ++it) { + if (ProcessMask(it->Logical(), pFilter, str)) { + aux_map[str] = true; + } + } + } else { + aux_map[std::string("default speaker")] = true; + } + + for (map_it = aux_map.begin(); map_it != aux_map.end(); ++map_it) { + rOut.push_back(map_it->first); + } + } + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + ParmKind2Str(unsigned parmKind, char *pOutString) + { + // :KLUDGE: Absolutely no idea what this is... + if ((parmKind & 0x003F) >= sizeof(mpParmKindNames)/sizeof(mpParmKindNames[0])) + return 0; + + strcpy(pOutString, mpParmKindNames[parmKind & 0x003F]); + + if (parmKind & PARAMKIND_E) strcat(pOutString, "_E"); + if (parmKind & PARAMKIND_N) strcat(pOutString, "_N"); + if (parmKind & PARAMKIND_D) strcat(pOutString, "_D"); + if (parmKind & PARAMKIND_A) strcat(pOutString, "_A"); + if (parmKind & PARAMKIND_C) strcat(pOutString, "_C"); + if (parmKind & PARAMKIND_Z) strcat(pOutString, "_Z"); + if (parmKind & PARAMKIND_K) strcat(pOutString, "_K"); + if (parmKind & PARAMKIND_0) strcat(pOutString, "_0"); + if (parmKind & PARAMKIND_V) strcat(pOutString, "_V"); + if (parmKind & PARAMKIND_T) strcat(pOutString, "_T"); + + return 1; + } + + + // //*************************************************************************** + // //*************************************************************************** + // void + // AddFileListToFeatureRepositories( + // const char* pFileName, + // const char* pFilter, + // std::queue<FeatureRepository *> &featureRepositoryList) + // { + // IStkStream l_stream; + // std::string file_name; + // Tokenizer file_list(pFileName, ","); + // Tokenizer::iterator p_file_name; + + // //:TODO: error if empty featureRepositoryList + // + // for (p_file_name = file_list.begin(); p_file_name != file_list.end(); ++p_file_name) + // { + // // get rid of initial and trailing blanks + // Trim(*p_file_name); + + // // open file name + // l_stream.open(p_file_name->c_str(), std::ios::in, pFilter); + // + // if (!l_stream.good()) { + // //:TODO: + // // Warning or error ... Why warning? -Lukas + // throw std::runtime_error(std::string("Cannot not open list file ") + + // *p_file_name); + // } + + // // read all lines and parse them + // for(;;) + // { + // l_stream >> file_name; + // //:TODO: if(l_stream.badl()) Error() + // // Reading after last token set the fail bit + // if(l_stream.fail()) + // break; + // // we can push_back a std::string as new FileListElem object + // // is created using FileListElem(const std::string&) constructor + // // and logical and physical names are correctly extracted + // featureRepositoryList.front()->mInputQueue.push_back(file_name); + // + // //cycle in the featureRepositoryList + // featureRepositoryList.push(featureRepositoryList.front()); + // featureRepositoryList.pop(); + // } + // l_stream.close(); + // } + // } // AddFileList(const std::string & rFileName) + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + Init( + bool swap, + int extLeft, + int extRight, + int targetKind, + int derivOrder, + int* pDerivWinLen, + const char* pCmnPath, + const char* pCmnMask, + const char* pCvnPath, + const char* pCvnMask, + const char* pCvgFile) + { + mSwapFeatures = swap; + mStartFrameExt = extLeft; + mEndFrameExt = extRight; + mTargetKind = targetKind; + mDerivOrder = derivOrder; + mDerivWinLengths = pDerivWinLen; + mpCmnPath = pCmnPath; + mpCmnMask = pCmnMask; + mpCvnPath = pCvnPath; + mpCvnMask = pCvnMask; + mpCvgFile = pCvgFile; + } // Init() + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + AddFile(const std::string & rFileName) + { + mInputQueue.push_back(rFileName); + } // AddFile(const std::string & rFileName) + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + AddFileList(const char* pFileName, const char* pFilter) + { + IStkStream l_stream; + std::string file_name; + Tokenizer file_list(pFileName, ","); + Tokenizer::iterator p_file_name; + + for (p_file_name = file_list.begin(); p_file_name != file_list.end(); ++p_file_name) + { + // get rid of spaces + Trim(*p_file_name); + + // open the file + l_stream.open(p_file_name->c_str(), std::ios::in, pFilter); + + if (!l_stream.good()) + { + //:TODO: + // Warning or error ... Why warning? -Lukas + throw std::runtime_error(std::string("Cannot not open list file ") + + *p_file_name); + } + // read all lines and parse them + for(;;) + { + l_stream >> file_name; + //:TODO: if(l_stream.badl()) Error() + // Reading after last token set the fail bit + if(l_stream.fail()) + break; + // we can push_back a std::string as new FileListElem object + // is created using FileListElem(const std::string&) constructor + // and logical and physical names are correctly extracted + mInputQueue.push_back(file_name); + } + l_stream.close(); + } + } // AddFileList(const std::string & rFileName) + + + //*************************************************************************** + //*************************************************************************** + void + FeatureRepository:: + MoveNext() + { + assert (mInputQueueIterator != mInputQueue.end()); + mInputQueueIterator++; + } // ReadFullMatrix(Matrix<BaseFloat>& rMatrix) + + + //*************************************************************************** + //*************************************************************************** + bool + FeatureRepository:: + ReadFullMatrix(Matrix<BaseFloat>& rMatrix) + { + // clear the matrix + rMatrix.Destroy(); + + // extract index file name + if (!mCurrentIndexFileDir.empty()) + { + char tmp_name[mCurrentIndexFileDir.length() + + mCurrentIndexFileExt.length() + + mInputQueueIterator->Physical().length()]; + + MakeHtkFileName(tmp_name, mInputQueueIterator->Physical().c_str(), + mCurrentIndexFileDir.c_str(), mCurrentIndexFileExt.c_str()); + + mCurrentIndexFileName = tmp_name; + } + else + mCurrentIndexFileName = ""; + + //get the 3-letter suffix + int pos_last_three_chars = mInputQueueIterator->Physical().size() - 3; + if (pos_last_three_chars < 0) pos_last_three_chars = 0; + //read the gzipped ascii features + if (mInputQueueIterator->Physical().substr(pos_last_three_chars) == ".gz") { + return ReadGzipAsciiFeatures(*mInputQueueIterator, rMatrix); + } + + // read the matrix and return the result + return ReadHTKFeatures(*mInputQueueIterator, rMatrix); + } // ReadFullMatrix(Matrix<BaseFloat>& rMatrix) + + + + //*************************************************************************** + //*************************************************************************** + bool + FeatureRepository:: + WriteFeatureMatrix(const Matrix<BaseFloat>& rMatrix, const std::string& filename, int targetKind, int samplePeriod) + { + FILE* fp = fopen(filename.c_str(),"w"); + if(NULL == fp) { Error(std::string("Cannot create file:") + filename); return false; } + + WriteHTKFeatures(fp, samplePeriod, targetKind, mSwapFeatures, const_cast<Matrix<BaseFloat>&>(rMatrix)); + + fclose(fp); + + return true; + } + + + //*************************************************************************** + //*************************************************************************** + // private: + int + FeatureRepository:: + ReadHTKHeader() + { + // TODO + // Change this... We should read from StkStream + FILE* fp = mStream.fp(); + + if (!fread(&mHeader.mNSamples, sizeof(INT_32), 1, fp)) return -1; + if (!fread(&mHeader.mSamplePeriod, sizeof(INT_32), 1, fp)) return -1; + if (!fread(&mHeader.mSampleSize, sizeof(INT_16), 1, fp)) return -1; + if (!fread(&mHeader.mSampleKind, sizeof(UINT_16), 1, fp)) return -1; + + if (mSwapFeatures) + { + swap4(mHeader.mNSamples); + swap4(mHeader.mSamplePeriod); + swap2(mHeader.mSampleSize); + swap2(mHeader.mSampleKind); + } + + if (mHeader.mSamplePeriod < 0 + || mHeader.mSamplePeriod > 1000000 + || mHeader.mNSamples < 0 + || mHeader.mSampleSize < 0) + { + return -1; + } + + return 0; + } + + + //*************************************************************************** + //*************************************************************************** + // private: + int + FeatureRepository:: + ReadHTKFeature( + BaseFloat* pIn, + size_t feaLen, + bool decompress, + BaseFloat* pScale, + BaseFloat* pBias) + { + FILE* fp = mStream.fp(); + + size_t i; + + if (decompress) + { + INT_16 s; + // BaseFloat pScale = (xmax - xmin) / (2*32767); + // BaseFloat pBias = (xmax + xmin) / 2; + + for (i = 0; i < feaLen; i++) + { + if (fread(&s, sizeof(INT_16), 1, fp) != 1) + return -1; + + if (mSwapFeatures) swap2(s); + pIn[i] = ((BaseFloat)s + pBias[i]) / pScale[i]; + } + + return 0; + } + +#if !DOUBLEPRECISION + if (fread(pIn, sizeof(FLOAT_32), feaLen, fp) != feaLen) + return -1; + + if (mSwapFeatures) + for (i = 0; i < feaLen; i++) + swap4(pIn[i]); +#else + float f; + + for (i = 0; i < feaLen; i++) + { + if (fread(&f, sizeof(FLOAT_32), 1, fp) != 1) + return -1; + + if (mSwapFeatures) + swap4(f); + + pIn[i] = f; + } +#endif + return 0; + } // int ReadHTKFeature + + + + //*************************************************************************** + //*************************************************************************** +/* bool + FeatureRepository:: + ReadHTKFeatures(const std::string& rFileName, Matrix<BaseFloat>& rFeatureMatrix) + { + std::string file_name(rFileName); + std::string cmn_file_name; + std::string cvn_file_name; + + int ext_left = mStartFrameExt; + int ext_right = mEndFrameExt; + int from_frame; + int to_frame; + int tot_frames; + int trg_vec_size; + int src_vec_size; + int src_deriv_order; + int lo_src_tgz_deriv_order; + int i; + int j; + int k; + int e; + int coefs; + int trg_E; + int trg_0; + int trg_N; + int src_E; + int src_0; + int src_N; + int comp; + int coef_size; + char* chptr; + + + + // read frame range definition if any ( physical_file.fea[s,e] ) + if ((chptr = strrchr(file_name.c_str(), '[')) == NULL || + ((i=0), sscanf(chptr, "[%d,%d]%n", &from_frame, &to_frame, &i), + chptr[i] != '\0')) + { + chptr = NULL; + } + + if (chptr != NULL) + *chptr = '\0'; + + // Experimental changes... + // if ((strcmp(file_name.c_str(), "-")) + // && (mpLastFileName != NULL) + // && (!strcmp(mpLastFileName, file_name.c_str()))) + // { + // mHeader = mLastHeader; + // } + // else + // { + // if (mpLastFileName) + // { + // //if (mpFp != stdin) + // // fclose(mpFp); + // mStream.close(); + // + // free(mpLastFileName); + // mpLastFileName = NULL; + // } + + if ((file_name != "-" ) + && (!mLastFileName.empty()) + && (mLastFileName == file_name)) + { + mHeader = mLastHeader; + } + else + { + if (!mLastFileName.empty()) + { + mStream.close(); + mLastFileName = ""; + } + + + // open the feature file + mStream.open(file_name.c_str(), ios::binary); + if (!mStream.good()) + { + Error("Cannot open feature file: '%s'", file_name.c_str()); + } + + + if (ReadHTKHeader()) + Error("Invalid HTK header in feature file: '%s'", file_name.c_str()); + + if (mHeader.mSampleKind & PARAMKIND_C) + { + // File is in compressed form, scale and pBias vectors + // are appended after HTK header. + + int coefs = mHeader.mSampleSize/sizeof(INT_16); + mpA = (BaseFloat*) realloc(mpA, coefs * sizeof(BaseFloat)); + mpB = (BaseFloat*) realloc(mpB, coefs * sizeof(BaseFloat)); + if (mpA == NULL || mpB == NULL) Error("Insufficient memory"); + + e = ReadHTKFeature(mpA, coefs, 0, 0, 0); + e |= ReadHTKFeature(mpB, coefs, 0, 0, 0); + + if (e) + Error("Cannot read feature file: '%s'", file_name.c_str()); + + mHeader.mNSamples -= 2 * sizeof(FLOAT_32) / sizeof(INT_16); + } + + // remember current settings + mLastFileName = file_name; + mLastHeader = mHeader; + } + + if (chptr != NULL) + *chptr = '['; + + if (chptr == NULL) + { // Range [s,e] was not specified + from_frame = 0; + to_frame = mHeader.mNSamples-1; + } + + src_deriv_order = PARAMKIND_T & mHeader.mSampleKind ? 3 : + PARAMKIND_A & mHeader.mSampleKind ? 2 : + PARAMKIND_D & mHeader.mSampleKind ? 1 : 0; + src_E = (PARAMKIND_E & mHeader.mSampleKind) != 0; + src_0 = (PARAMKIND_0 & mHeader.mSampleKind) != 0; + src_N = ((PARAMKIND_N & mHeader.mSampleKind) != 0) * (src_E + src_0); + comp = PARAMKIND_C & mHeader.mSampleKind; + + mHeader.mSampleKind &= ~PARAMKIND_C; + + if (mTargetKind == PARAMKIND_ANON) + { + mTargetKind = mHeader.mSampleKind; + } + else if ((mTargetKind & 077) == PARAMKIND_ANON) + { + mTargetKind &= ~077; + mTargetKind |= mHeader.mSampleKind & 077; + } + + trg_E = (PARAMKIND_E & mTargetKind) != 0; + trg_0 = (PARAMKIND_0 & mTargetKind) != 0; + trg_N =((PARAMKIND_N & mTargetKind) != 0) * (trg_E + trg_0); + + coef_size = comp ? sizeof(INT_16) : sizeof(FLOAT_32); + coefs = (mHeader.mSampleSize/coef_size + src_N) / + (src_deriv_order+1) - src_E - src_0; + src_vec_size = (coefs + src_E + src_0) * (src_deriv_order+1) - src_N; + + //Is coefs dividable by 1 + number of derivatives specified in header + if (src_vec_size * coef_size != mHeader.mSampleSize) + { + Error("Invalid HTK header in feature file: '%s'. " + "mSampleSize do not match with parmKind", file_name.c_str()); + } + + if (mDerivOrder < 0) + mDerivOrder = src_deriv_order; + + + if ((!src_E && trg_E) || (!src_0 && trg_0) || (src_N && !trg_N) || + (trg_N && !trg_E && !trg_0) || (trg_N && !mDerivOrder) || + (src_N && !src_deriv_order && mDerivOrder) || + ((mHeader.mSampleKind & 077) != (mTargetKind & 077) && + (mHeader.mSampleKind & 077) != PARAMKIND_ANON)) + { + char srcParmKind[64]; + char trgParmKind[64]; + + ParmKind2Str(mHeader.mSampleKind, srcParmKind); + ParmKind2Str(mTargetKind, trgParmKind); + Error("Cannot convert %s to %s", srcParmKind, trgParmKind); + } + + lo_src_tgz_deriv_order = LOWER_OF(src_deriv_order, mDerivOrder); + trg_vec_size = (coefs + trg_E + trg_0) * (mDerivOrder+1) - trg_N; + + i = LOWER_OF(from_frame, mStartFrameExt); + from_frame -= i; + ext_left -= i; + + i = LOWER_OF(mHeader.mNSamples-to_frame-1, mEndFrameExt); + to_frame += i; + ext_right -= i; + + if (from_frame > to_frame || from_frame >= mHeader.mNSamples || to_frame< 0) + Error("Invalid frame range for feature file: '%s'", file_name.c_str()); + + tot_frames = to_frame - from_frame + 1 + ext_left + ext_right; + + // initialize matrix + rFeatureMatrix.Init(tot_frames, trg_vec_size); + + // fill the matrix with features + for (i = 0; i <= to_frame - from_frame; i++) + { + BaseFloat* A = mpA; + BaseFloat* B = mpB; + BaseFloat* mxPtr = rFeatureMatrix[i+ext_left]; + + // seek to the desired position + fseek(mStream.fp(), + sizeof(HtkHeader) + (comp ? src_vec_size * 2 * sizeof(FLOAT_32) : 0) + + (from_frame + i) * src_vec_size * coef_size, + SEEK_SET); + + e = ReadHTKFeature(mxPtr, coefs, comp, A, B); + + mxPtr += coefs; + A += coefs; + B += coefs; + + if (src_0 && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_0 && !trg_N) mxPtr++; + if (src_E && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_E && !trg_N) mxPtr++; + + for (j = 0; j < lo_src_tgz_deriv_order; j++) + { + e |= ReadHTKFeature(mxPtr, coefs, comp, A, B); + mxPtr += coefs; + A += coefs; + B += coefs; + + if (src_0) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_0) mxPtr++; + if (src_E) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_E) mxPtr++; + } + + if (e) + Error("Cannot read feature file: '%s' frame %d/%d", file_name.c_str(), + i, to_frame - from_frame + 1); + } + + // From now, coefs includes also trg_0 + trg_E ! + coefs += trg_0 + trg_E; + + // If extension of the matrix to the left or to the right is required, + // perform it here + for (i = 0; i < ext_left; i++) + { + memcpy(rFeatureMatrix[i], + rFeatureMatrix[ext_left], + (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat)); + } + + for (i = tot_frames - ext_right; i < tot_frames; i++) + { + memcpy(rFeatureMatrix[i], + rFeatureMatrix[tot_frames - ext_right - 1], + (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat)); + } + + // Sentence cepstral mean normalization + if( (mpCmnPath == NULL) + && !(PARAMKIND_Z & mHeader.mSampleKind) + && (PARAMKIND_Z & mTargetKind)) + { + // for each coefficient + for(j=0; j < coefs; j++) + { + BaseFloat norm = 0.0; + for(i=0; i < tot_frames; i++) // for each frame + { + norm += rFeatureMatrix[i][j - trg_N]; + //norm += fea_mx[i*trg_vec_size - trg_N + j]; + } + + norm /= tot_frames; + + for(i=0; i < tot_frames; i++) // for each frame + rFeatureMatrix[i][j - trg_N] -= norm; + //fea_mx[i*trg_vec_size - trg_N + j] -= norm; + } + } + + // Compute missing derivatives + for (; src_deriv_order < mDerivOrder; src_deriv_order++) + { + int winLen = mDerivWinLengths[src_deriv_order]; + BaseFloat norm = 0.0; + + for (k = 1; k <= winLen; k++) + { + norm += 2 * k * k; + } + + // for each frame + for (i=0; i < tot_frames; i++) + { + // for each coefficient + for (j=0; j < coefs; j++) + { + //BaseFloat* src = fea_mx + i*trg_vec_size + src_deriv_order*coefs - trg_N + j; + BaseFloat* src = &rFeatureMatrix[i][src_deriv_order*coefs - trg_N + j]; + + *(src + coefs) = 0.0; + + if (i < winLen || i >= tot_frames-winLen) + { // boundaries need special treatment + for (k = 1; k <= winLen; k++) + { + *(src+coefs) += k*(src[ LOWER_OF(tot_frames-1-i,k)*rFeatureMatrix.Stride()] + -src[-LOWER_OF(i, k)*rFeatureMatrix.Stride()]); + } + } + else + { // otherwise use more efficient code + for (k = 1; k <= winLen; k++) + { + *(src+coefs) += k*(src[ k * rFeatureMatrix.Stride()] + -src[-k * rFeatureMatrix.Stride()]); + } + } + *(src + coefs) /= norm; + } + } + } + + mHeader.mNSamples = tot_frames; + mHeader.mSampleSize = trg_vec_size * sizeof(FLOAT_32); + mHeader.mSampleKind = mTargetKind & ~(PARAMKIND_D | PARAMKIND_A | PARAMKIND_T); + + + //////////////////////////////////////////////////////////////////////////// + /////////////// Cepstral mean and variance normalization /////////////////// + //////////////////////////////////////////////////////////////////////////// + //......................................................................... + if (mpCmnPath != NULL + && mpCmnMask != NULL) + { + // retrieve file name + ProcessMask(file_name, mpCmnMask, cmn_file_name); + // add the path correctly + cmn_file_name.insert(0, "/"); + cmn_file_name.insert(0, mpCmnPath); + + // read the file + ReadCepsNormFile(cmn_file_name.c_str(), &mpLastCmnFile, &mpCmn, + mHeader.mSampleKind & ~PARAMKIND_Z, CNF_Mean, coefs); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < coefs; j++) + { + rFeatureMatrix[i][j - trg_N] -= mpCmn[j]; + } + } + } + + mHeader.mSampleKind |= mDerivOrder==3 ? PARAMKIND_D | PARAMKIND_A | PARAMKIND_T : + mDerivOrder==2 ? PARAMKIND_D | PARAMKIND_A : + mDerivOrder==1 ? PARAMKIND_D : 0; + + //......................................................................... + if (mpCvnPath != NULL + && mpCvnMask != NULL) + { + // retrieve file name + ProcessMask(file_name, mpCvnMask, cvn_file_name); + // add the path correctly + cvn_file_name.insert(0, "/"); + cvn_file_name.insert(0, mpCvnPath); + + // read the file + ReadCepsNormFile(cvn_file_name.c_str(), &mpLastCvnFile, &mpCvn, + mHeader.mSampleKind, CNF_Variance, trg_vec_size); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < trg_vec_size; j++) + { + rFeatureMatrix[i][j - trg_N] *= mpCvn[j]; + } + } + } + + //......................................................................... + // process the global covariance file + if (mpCvgFile != NULL) + { + ReadCepsNormFile(mpCvgFile, &mpLastCvgFile, &mpCvg, + -1, CNF_VarScale, trg_vec_size); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < trg_vec_size; j++) + { + rFeatureMatrix[i][j - trg_N] *= mpCvg[j]; + } + } + } + + return true; + } +*/ + + //*************************************************************************** + //*************************************************************************** + + + + + + //*************************************************************************** + //*************************************************************************** + bool + FeatureRepository:: + ReadHTKFeatures(const FileListElem& rFileNameRecord, + Matrix<BaseFloat>& rFeatureMatrix) + { + std::string file_name(rFileNameRecord.Physical()); + std::string cmn_file_name; + std::string cvn_file_name; + + int ext_left = mStartFrameExt; + int ext_right = mEndFrameExt; + int from_frame; + int to_frame; + int tot_frames; + int trg_vec_size; + int src_vec_size; + int src_deriv_order; + int lo_src_tgz_deriv_order; + int i; + int j; + int k; + int e; + int coefs; + int trg_E; + int trg_0; + int trg_N; + int src_E; + int src_0; + int src_N; + int comp; + int coef_size; + char* chptr; + + + TIMER_START(mTim); + + // read frame range definition if any ( physical_file.fea[s,e] ) + if ((chptr = strrchr((char*)file_name.c_str(), '[')) == NULL || + ((i=0), sscanf(chptr, "[%d,%d]%n", &from_frame, &to_frame, &i), + chptr[i] != '\0')) + { + chptr = NULL; + } + + if (chptr != NULL) + *chptr = '\0'; + + + if ((file_name != "-" ) + && (!mLastFileName.empty()) + && (mLastFileName == file_name)) + { + mHeader = mLastHeader; + } + else + { + if (!mLastFileName.empty()) + { + mStream.close(); + mLastFileName = ""; + } + + + // open the feature file + mStream.open(file_name.c_str(), std::ios::binary); + if (!mStream.good()) + { + throw std::runtime_error(std::string("Cannot open feature file: '") + + file_name.c_str() + "'"); + } + + + if (ReadHTKHeader()) { + throw std::runtime_error(std::string("Invalid HTK header in feature file: '") + + file_name.c_str() + "'"); + } + + if (mHeader.mSampleKind & PARAMKIND_C) + { + // File is in compressed form, scale and pBias vectors + // are appended after HTK header. + coefs = mHeader.mSampleSize/sizeof(INT_16); + + mpA = (BaseFloat*) realloc(mpA, coefs * sizeof(BaseFloat)); + mpB = (BaseFloat*) realloc(mpB, coefs * sizeof(BaseFloat)); + + if (mpA == NULL || mpB == NULL) { + throw std::runtime_error("Insufficient memory"); + } + + e = ReadHTKFeature(mpA, coefs, 0, 0, 0); + e |= ReadHTKFeature(mpB, coefs, 0, 0, 0); + + if (e) { + throw std::runtime_error(std::string("Cannot read feature file: '") + + file_name.c_str() + "'"); + } + + mHeader.mNSamples -= 2 * sizeof(FLOAT_32) / sizeof(INT_16); + } + + // remember current settings + mLastFileName = file_name; + mLastHeader = mHeader; + } + + if (chptr != NULL) { + *chptr = '['; + } + + if (chptr == NULL) { + // Range [s,e] was not specified + from_frame = 0; + to_frame = mHeader.mNSamples-1; + } + + src_deriv_order = PARAMKIND_T & mHeader.mSampleKind ? 3 : + PARAMKIND_A & mHeader.mSampleKind ? 2 : + PARAMKIND_D & mHeader.mSampleKind ? 1 : 0; + src_E = (PARAMKIND_E & mHeader.mSampleKind) != 0; + src_0 = (PARAMKIND_0 & mHeader.mSampleKind) != 0; + src_N = ((PARAMKIND_N & mHeader.mSampleKind) != 0) * (src_E + src_0); + comp = PARAMKIND_C & mHeader.mSampleKind; + + mHeader.mSampleKind &= ~PARAMKIND_C; + + if (mTargetKind == PARAMKIND_ANON) + { + mTargetKind = mHeader.mSampleKind; + } + else if ((mTargetKind & 077) == PARAMKIND_ANON) + { + mTargetKind &= ~077; + mTargetKind |= mHeader.mSampleKind & 077; + } + + trg_E = (PARAMKIND_E & mTargetKind) != 0; + trg_0 = (PARAMKIND_0 & mTargetKind) != 0; + trg_N =((PARAMKIND_N & mTargetKind) != 0) * (trg_E + trg_0); + + coef_size = comp ? sizeof(INT_16) : sizeof(FLOAT_32); + coefs = (mHeader.mSampleSize/coef_size + src_N) / + (src_deriv_order+1) - src_E - src_0; + src_vec_size = (coefs + src_E + src_0) * (src_deriv_order+1) - src_N; + + //Is coefs dividable by 1 + number of derivatives specified in header + if (src_vec_size * coef_size != mHeader.mSampleSize) + { + throw std::runtime_error(std::string("Invalid HTK header in feature file: '") + + file_name + "' mSampleSize do not match with parmKind"); + } + + if (mDerivOrder < 0) + mDerivOrder = src_deriv_order; + + + if ((!src_E && trg_E) || (!src_0 && trg_0) || (src_N && !trg_N) || + (trg_N && !trg_E && !trg_0) || (trg_N && !mDerivOrder) || + (src_N && !src_deriv_order && mDerivOrder) || + ((mHeader.mSampleKind & 077) != (mTargetKind & 077) && + (mHeader.mSampleKind & 077) != PARAMKIND_ANON)) + { + char srcParmKind[64]; + char trgParmKind[64]; + memset(srcParmKind,0,64); + memset(trgParmKind,0,64); + + ParmKind2Str(mHeader.mSampleKind, srcParmKind); + ParmKind2Str(mTargetKind, trgParmKind); + throw std::runtime_error(std::string("Cannot convert ") + srcParmKind + + " to " + trgParmKind); + } + + lo_src_tgz_deriv_order = std::min(src_deriv_order, mDerivOrder); + trg_vec_size = (coefs + trg_E + trg_0) * (mDerivOrder+1) - trg_N; + + i = std::min(from_frame, mStartFrameExt); + from_frame -= i; + ext_left -= i; + + i = std::min(mHeader.mNSamples-to_frame-1, mEndFrameExt); + to_frame += i; + ext_right -= i; + + if (from_frame > to_frame || from_frame >= mHeader.mNSamples || to_frame< 0) + throw std::runtime_error(std::string("Invalid frame range for feature file: '") + + file_name.c_str() + "'"); + + tot_frames = to_frame - from_frame + 1 + ext_left + ext_right; + + + TIMER_END(mTim,mTimeOpen); + + + // initialize matrix + rFeatureMatrix.Init(tot_frames, trg_vec_size, false); + + // fill the matrix with features + for (i = 0; i <= to_frame - from_frame; i++) + { + BaseFloat* A = mpA; + BaseFloat* B = mpB; + BaseFloat* mxPtr = rFeatureMatrix.pRowData(i+ext_left); + + TIMER_START(mTim); + // seek to the desired position + fseek(mStream.fp(), + sizeof(HtkHeader) + (comp ? src_vec_size * 2 * sizeof(FLOAT_32) : 0) + + (from_frame + i) * src_vec_size * coef_size, + SEEK_SET); + TIMER_END(mTim,mTimeSeek); + + TIMER_START(mTim); + // read + e = ReadHTKFeature(mxPtr, coefs, comp, A, B); + TIMER_END(mTim,mTimeRead); + + mxPtr += coefs; + A += coefs; + B += coefs; + + if (src_0 && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_0 && !trg_N) mxPtr++; + if (src_E && !src_N) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_E && !trg_N) mxPtr++; + + for (j = 0; j < lo_src_tgz_deriv_order; j++) + { + e |= ReadHTKFeature(mxPtr, coefs, comp, A, B); + mxPtr += coefs; + A += coefs; + B += coefs; + + if (src_0) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_0) mxPtr++; + if (src_E) e |= ReadHTKFeature(mxPtr, 1, comp, A++, B++); + if (trg_E) mxPtr++; + } + + if (e) { + std::cout << mHeader.mNSamples << "\n"; + std::cout << 2 * sizeof(FLOAT_32) / sizeof(INT_16) << "\n"; + std::cout << "from" << from_frame << "to" << to_frame << "i" << i << "\n"; + + std::ostringstream s; + s << i << "/" << to_frame - from_frame + 1, s.str(); + throw std::runtime_error(std::string("Cannot read feature file: '") + + file_name + "' frame " + s.str()); + } + } + + // From now, coefs includes also trg_0 + trg_E ! + coefs += trg_0 + trg_E; + + // If extension of the matrix to the left or to the right is required, + // perform it here + for (i = 0; i < ext_left; i++) + { + memcpy(rFeatureMatrix.pRowData(i), + rFeatureMatrix.pRowData(ext_left), + (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat)); + } + + for (i = tot_frames - ext_right; i < tot_frames; i++) + { + memcpy(rFeatureMatrix.pRowData(i), + rFeatureMatrix.pRowData(tot_frames - ext_right - 1), + (coefs * (1+lo_src_tgz_deriv_order) - trg_N) * sizeof(BaseFloat)); + } + + // Sentence cepstral mean normalization + if( (mpCmnPath == NULL) + && !(PARAMKIND_Z & mHeader.mSampleKind) + && (PARAMKIND_Z & mTargetKind)) + { + // for each coefficient + for(j=0; j < coefs; j++) + { + BaseFloat norm = 0.0; + for(i=0; i < tot_frames; i++) // for each frame + { + norm += rFeatureMatrix[i][j - trg_N]; + //norm += fea_mx[i*trg_vec_size - trg_N + j]; + } + + norm /= tot_frames; + + for(i=0; i < tot_frames; i++) // for each frame + rFeatureMatrix[i][j - trg_N] -= norm; + //fea_mx[i*trg_vec_size - trg_N + j] -= norm; + } + } + + // Compute missing derivatives + for (; src_deriv_order < mDerivOrder; src_deriv_order++) + { + int winLen = mDerivWinLengths[src_deriv_order]; + BaseFloat norm = 0.0; + + for (k = 1; k <= winLen; k++) + { + norm += 2 * k * k; + } + + // for each frame + for (i=0; i < tot_frames; i++) + { + // for each coefficient + for (j=0; j < coefs; j++) + { + //BaseFloat* src = fea_mx + i*trg_vec_size + src_deriv_order*coefs - trg_N + j; + BaseFloat* src = &rFeatureMatrix[i][src_deriv_order*coefs - trg_N + j]; + + *(src + coefs) = 0.0; + + if (i < winLen || i >= tot_frames-winLen) + { // boundaries need special treatment + for (k = 1; k <= winLen; k++) + { + *(src+coefs) += k*(src[ std::min(tot_frames-1-i,k)*rFeatureMatrix.Stride()] + -src[-std::min(i, k)*rFeatureMatrix.Stride()]); + } + } + else + { // otherwise use more efficient code + for (k = 1; k <= winLen; k++) + { + *(src+coefs) += k*(src[ k * rFeatureMatrix.Stride()] + -src[-k * rFeatureMatrix.Stride()]); + } + } + *(src + coefs) /= norm; + } + } + } + + mHeader.mNSamples = tot_frames; + mHeader.mSampleSize = trg_vec_size * sizeof(FLOAT_32); + mHeader.mSampleKind = mTargetKind & ~(PARAMKIND_D | PARAMKIND_A | PARAMKIND_T); + + + TIMER_START(mTim); + //////////////////////////////////////////////////////////////////////////// + /////////////// Cepstral mean and variance normalization /////////////////// + //////////////////////////////////////////////////////////////////////////// + //......................................................................... + if (mpCmnPath != NULL + && mpCmnMask != NULL) + { + // retrieve file name + ProcessMask(rFileNameRecord.Logical(), mpCmnMask, cmn_file_name); + // add the path correctly + + if(cmn_file_name == "") { + throw std::runtime_error("CMN Matching failed"); + } + + cmn_file_name.insert(0, "/"); + cmn_file_name.insert(0, mpCmnPath); + + // read the file + ReadCepsNormFile(cmn_file_name.c_str(), &mpLastCmnFile, &mpCmn, + mHeader.mSampleKind & ~PARAMKIND_Z, CNF_Mean, coefs); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < coefs; j++) + { + rFeatureMatrix[i][j - trg_N] -= mpCmn[j]; + } + } + } + + mHeader.mSampleKind |= mDerivOrder==3 ? PARAMKIND_D | PARAMKIND_A | PARAMKIND_T : + mDerivOrder==2 ? PARAMKIND_D | PARAMKIND_A : + mDerivOrder==1 ? PARAMKIND_D : 0; + + //......................................................................... + if (mpCvnPath != NULL + && mpCvnMask != NULL) + { + // retrieve file name + ProcessMask(rFileNameRecord.Logical(), mpCvnMask, cvn_file_name); + // add the path correctly + cvn_file_name.insert(0, "/"); + cvn_file_name.insert(0, mpCvnPath); + + // read the file + ReadCepsNormFile(cvn_file_name.c_str(), &mpLastCvnFile, &mpCvn, + mHeader.mSampleKind, CNF_Variance, trg_vec_size); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < trg_vec_size; j++) + { + rFeatureMatrix[i][j - trg_N] *= mpCvn[j]; + } + } + } + + //......................................................................... + // process the global covariance file + if (mpCvgFile != NULL) + { + ReadCepsNormFile(mpCvgFile, &mpLastCvgFile, &mpCvg, + -1, CNF_VarScale, trg_vec_size); + + // recompute feature values + for (i=0; i < tot_frames; i++) + { + for (j=trg_N; j < trg_vec_size; j++) + { + rFeatureMatrix[i][j - trg_N] *= mpCvg[j]; + } + } + } + + TIMER_END(mTim,mTimeNormalize); + + return true; + } + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + ReadParmKind(const char *str, bool checkBrackets) + { + unsigned int i; + int parmKind =0; + int slen = strlen(str); + + if (checkBrackets) + { + if (str[0] != '<' || str[slen-1] != '>') return -1; + str++; slen -= 2; + } + + for (; slen >= 0 && str[slen-2] == '_'; slen -= 2) + { + parmKind |= str[slen-1] == 'E' ? PARAMKIND_E : + str[slen-1] == 'N' ? PARAMKIND_N : + str[slen-1] == 'D' ? PARAMKIND_D : + str[slen-1] == 'A' ? PARAMKIND_A : + str[slen-1] == 'C' ? PARAMKIND_C : + str[slen-1] == 'Z' ? PARAMKIND_Z : + str[slen-1] == 'K' ? PARAMKIND_K : + str[slen-1] == '0' ? PARAMKIND_0 : + str[slen-1] == 'V' ? PARAMKIND_V : + str[slen-1] == 'T' ? PARAMKIND_T : -1; + + if (parmKind == -1) return -1; + } + + for (i = 0; i < sizeof(mpParmKindNames) / sizeof(char*); i++) + { + if (!strncmp(str, mpParmKindNames[i], slen)) + return parmKind | i; + } + return -1; + } + + + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + WriteHTKHeader (FILE * pOutFp, HtkHeader header, bool swap) + { + int cc; + + if (swap) { + swap4(header.mNSamples); + swap4(header.mSamplePeriod); + swap2(header.mSampleSize); + swap2(header.mSampleKind); + } + + fseek (pOutFp, 0L, SEEK_SET); + cc = fwrite(&header, sizeof(HtkHeader), 1, pOutFp); + + if (swap) { + swap4(header.mNSamples); + swap4(header.mSamplePeriod); + swap2(header.mSampleSize); + swap2(header.mSampleKind); + } + + return cc == 1 ? 0 : -1; + } + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + WriteHTKFeature( + FILE * pOutFp, + FLOAT * pOut, + size_t feaLen, + bool swap, + bool compress, + FLOAT* pScale, + FLOAT* pBias) + { + size_t i; + size_t cc = 0; + + + if (compress) + { + INT_16 s; + + for (i = 0; i < feaLen; i++) + { + s = pOut[i] * pScale[i] - pBias[i]; + if (swap) + swap2(s); + cc += fwrite(&s, sizeof(INT_16), 1, pOutFp); + } + + } else { + #if !DOUBLEPRECISION + if (swap) + for (i = 0; i < feaLen; i++) + swap4(pOut[i]); + + cc = fwrite(pOut, sizeof(FLOAT_32), feaLen, pOutFp); + + if (swap) + for (i = 0; i < feaLen; i++) + swap4(pOut[i]); + #else + FLOAT_32 f; + + for (i = 0; i < feaLen; i++) + { + f = pOut[i]; + if (swap) + swap4(f); + cc += fwrite(&f, sizeof(FLOAT_32), 1, pOutFp); + } + #endif + } + return cc == feaLen ? 0 : -1; + } + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + WriteHTKFeatures( + FILE * pOutFp, + FLOAT * pOut, + int nCoeffs, + int nSamples, + int samplePeriod, + int targetKind, + bool swap) + { + HtkHeader header; + int i, j; + FLOAT *pScale = NULL; + FLOAT *pBias = NULL; + + header.mNSamples = nSamples + ((targetKind & PARAMKIND_C) ? 2 * sizeof(FLOAT_32) / sizeof(INT_16) : 0); + header.mSamplePeriod = samplePeriod; + header.mSampleSize = nCoeffs * ((targetKind & PARAMKIND_C) ? sizeof(INT_16) : sizeof(FLOAT_32));; + header.mSampleKind = targetKind; + + WriteHTKHeader (pOutFp, header, swap); + + if(targetKind & PARAMKIND_C) { + pScale = (FLOAT*) malloc(nCoeffs * sizeof(FLOAT)); + pBias = (FLOAT*) malloc(nCoeffs * sizeof(FLOAT)); + if (pScale == NULL || pBias == NULL) Error("Insufficient memory"); + + for(i = 0; i < nCoeffs; i++) { + float xmin, xmax; + xmin = xmax = pOut[i]; + for(j = 1; j < nSamples; j++) { + if(pOut[j*nCoeffs+i] > xmax) xmax = pOut[j*nCoeffs+i]; + if(pOut[j*nCoeffs+i] < xmin) xmin = pOut[j*nCoeffs+i]; + } + pScale[i] = (2*32767) / (xmax - xmin); + pBias[i] = pScale[i] * (xmax + xmin) / 2; + + + } + if (WriteHTKFeature(pOutFp, pScale, nCoeffs, swap, false, 0, 0) + || WriteHTKFeature(pOutFp, pBias, nCoeffs, swap, false, 0, 0)) { + return -1; + } + } + for(j = 0; j < nSamples; j++) { + if (WriteHTKFeature(pOutFp, &pOut[j*nCoeffs], nCoeffs, swap, targetKind & PARAMKIND_C, pScale, pBias)) { + return -1; + } + } + return 0; + } + + + //*************************************************************************** + //*************************************************************************** + int + FeatureRepository:: + WriteHTKFeatures( + FILE * pOutFp, + int samplePeriod, + int targetKind, + bool swap, + Matrix<BaseFloat>& rFeatureMatrix) + { + HtkHeader header; + size_t i, j; + FLOAT *p_scale = NULL; + FLOAT *p_bias = NULL; + size_t n_samples = rFeatureMatrix.Rows(); + size_t n_coeffs = rFeatureMatrix.Cols(); + + header.mNSamples = n_samples + ((targetKind & PARAMKIND_C) ? 2 * sizeof(FLOAT_32) / sizeof(INT_16) : 0); + header.mSamplePeriod = samplePeriod; + header.mSampleSize = n_coeffs * ((targetKind & PARAMKIND_C) ? sizeof(INT_16) : sizeof(FLOAT_32));; + header.mSampleKind = targetKind; + + WriteHTKHeader (pOutFp, header, swap); + + if(targetKind & PARAMKIND_C) { + p_scale = (FLOAT*) malloc(n_coeffs * sizeof(FLOAT)); + p_bias = (FLOAT*) malloc(n_coeffs * sizeof(FLOAT)); + if (p_scale == NULL || p_bias == NULL) Error("Insufficient memory"); + + for(i = 0; i < n_coeffs; i++) { + float xmin, xmax; + xmin = xmax = rFeatureMatrix[0][i]; + + for(j = 1; j < n_samples; j++) { + if(rFeatureMatrix[j][i] > xmax) xmax = rFeatureMatrix[j][i]; + if(rFeatureMatrix[j][i] < xmin) xmin = rFeatureMatrix[j][i]; + } + + p_scale[i] = (2*32767) / (xmax - xmin); + p_bias[i] = p_scale[i] * (xmax + xmin) / 2; + } + + if (WriteHTKFeature(pOutFp, p_scale, n_coeffs, swap, false, 0, 0) + || WriteHTKFeature(pOutFp, p_bias, n_coeffs, swap, false, 0, 0)) { + return -1; + } + } + + for(j = 0; j < n_samples; j++) { + if (WriteHTKFeature(pOutFp, rFeatureMatrix[j].pData(), n_coeffs, swap, targetKind & PARAMKIND_C, p_scale, p_bias)) { + return -1; + } + } + + return 0; + } + + //*************************************************************************** + //*************************************************************************** + + + bool + FeatureRepository:: + ReadGzipAsciiFeatures(const FileListElem& rFileNameRecord, Matrix<BaseFloat>& rFeatureMatrix) + { + //build the command + std::string cmd("gunzip -c "); cmd += rFileNameRecord.Physical(); + + //define buffer + const int buf_size=262144; + char buf[buf_size]; + char vbuf[2*buf_size]; + + TIMER_START(mTim); + //open the pipe + FILE* fp = popen(cmd.c_str(),"r"); + if(fp == NULL) { + //2nd try... + Warning(std::string("2nd try to open pipe: ")+cmd); + sleep(5); + fp = popen(cmd.c_str(),"r"); + if(fp == NULL) { + KALDI_ERR << "Cannot open pipe: " << cmd; + } + } + setvbuf(fp,vbuf,_IOFBF,2*buf_size); + TIMER_END(mTim,mTimeOpen); + + //string will stay allocated across calls + static std::string line; line.resize(0); + + //define matrix storage + static int cols = 131072; + std::list<std::vector<BaseFloat> > matrix(1); + matrix.front().reserve(cols); + + //read all the lines to a vector + int line_ctr=1; + while(1) { + TIMER_START(mTim); + if(NULL == fgets(buf,buf_size,fp)) break; + TIMER_END(mTim,mTimeRead); + + line += buf; + if(*(line.rbegin()) == '\n' || feof(fp)) { + //parse the line of numbers + TIMER_START(mTim); + const char* ptr = line.c_str(); + char* end; + while(1) { + //skip whitespace + while(isspace(*ptr)) ptr++; + if(*ptr == 0) break; + //check that a number follows + switch(*ptr) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case '.': case '+': case '-': + break; + default : KALDI_ERR << "A number was expected:" << ptr + << " reading from" << cmd; + exit(1); + } + //read a number + BaseFloat val = strtof(ptr,&end); ptr=end; + matrix.back().push_back(val); + } + TIMER_END(mTim,mTimeNormalize); + //we have the line of numbers, insert empty row to matrix + if(matrix.back().size() > 0 && !feof(fp)) { + matrix.push_back(std::vector<BaseFloat>()); + matrix.back().reserve(matrix.front().size()); + } + //dispose the current line + line.resize(0);//but stay allocated... + line_ctr++; + } + } + if(matrix.back().size() == 0) matrix.pop_back(); + + //get matrix dimensions + int rows = matrix.size(); + /*int*/ cols = matrix.front().size(); + + //define interators + std::list<std::vector<BaseFloat> >::iterator it_r; + std::vector<BaseFloat>::iterator it_c; + + //check that all lines have same size + int i; + for(i=0,it_r=matrix.begin(); it_r != matrix.end(); ++i,++it_r) { + if(it_r->size() != cols) { + KALDI_ERR << "All rows must have same dimension, 1st line cols: " << cols + << ", " << i << "th line cols: " << it_r->size(); + } + } + + //copy data to matrix + TIMER_START(mTim); + rFeatureMatrix.Init(rows,cols); + int r,c; + for(r=0,it_r=matrix.begin(); it_r!=matrix.end(); ++r,++it_r) { + for(c=0,it_c=it_r->begin(); it_c!=it_r->end(); ++c,++it_c) { + rFeatureMatrix(r,c) = *it_c; + } + } + TIMER_END(mTim,mTimeSeek); + + //close the pipe + if(pclose(fp) == -1) { + KALDI_ERR << "Cannot close pipe: " << cmd; + } + + return true; + } + + + //*************************************************************************** + //*************************************************************************** + +} // namespace TNet |