diff options
Diffstat (limited to 'kaldi_io/src/kaldi/util/kaldi-table-inl.h')
| -rw-r--r-- | kaldi_io/src/kaldi/util/kaldi-table-inl.h | 2246 |
1 files changed, 0 insertions, 2246 deletions
diff --git a/kaldi_io/src/kaldi/util/kaldi-table-inl.h b/kaldi_io/src/kaldi/util/kaldi-table-inl.h deleted file mode 100644 index 6b73c88..0000000 --- a/kaldi_io/src/kaldi/util/kaldi-table-inl.h +++ /dev/null @@ -1,2246 +0,0 @@ -// util/kaldi-table-inl.h - -// Copyright 2009-2011 Microsoft Corporation -// 2013 Johns Hopkins University (author: Daniel Povey) - -// See ../../COPYING for clarification regarding multiple authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED -// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// See the Apache 2 License for the specific language governing permissions and -// limitations under the License. - - -#ifndef KALDI_UTIL_KALDI_TABLE_INL_H_ -#define KALDI_UTIL_KALDI_TABLE_INL_H_ - -#include <algorithm> -#include "util/kaldi-io.h" -#include "util/text-utils.h" -#include "util/stl-utils.h" // for StringHasher. - - -namespace kaldi { - -/// \addtogroup table_impl_types -/// @{ - -template<class Holder> class SequentialTableReaderImplBase { - public: - typedef typename Holder::T T; - // note that Open takes rxfilename not rspecifier. - virtual bool Open(const std::string &rxfilename) = 0; - virtual bool Done() const = 0; - virtual bool IsOpen() const = 0; - virtual std::string Key() = 0; - virtual const T &Value() = 0; - virtual void FreeCurrent() = 0; - virtual void Next() = 0; - virtual bool Close() = 0; - SequentialTableReaderImplBase() { } - virtual ~SequentialTableReaderImplBase() { } - private: - KALDI_DISALLOW_COPY_AND_ASSIGN(SequentialTableReaderImplBase); -}; - - -// This is the implementation for SequentialTableReader -// when it's actually a script file. -template<class Holder> class SequentialTableReaderScriptImpl: - public SequentialTableReaderImplBase<Holder> { - public: - typedef typename Holder::T T; - - SequentialTableReaderScriptImpl(): state_(kUninitialized) { } - - virtual bool Open(const std::string &rspecifier) { - if (state_ != kUninitialized) - if (! Close()) // call Close() yourself to suppress this exception. - KALDI_ERR << "TableReader::Open, error closing previous input: " - << "rspecifier was " << rspecifier_; - bool binary; - rspecifier_ = rspecifier; - RspecifierType rs = ClassifyRspecifier(rspecifier, &script_rxfilename_, - &opts_); - KALDI_ASSERT(rs == kScriptRspecifier); - if (!script_input_.Open(script_rxfilename_, &binary)) { // Failure on Open - KALDI_WARN << "Failed to open script file " - << PrintableRxfilename(script_rxfilename_); - state_ = kUninitialized; - return false; - } else { // Open succeeded. - if (binary) { // script file should not be binary file.. - state_ = kError; // bad script file. - script_input_.Close(); - return false; - } else { - state_ = kFileStart; - Next(); - if (state_ == kError) { - script_input_.Close(); - return false; - } - if (opts_.permissive) { // Next() will have preloaded. - KALDI_ASSERT(state_ == kLoadSucceeded || state_ == kEof); - } else { - KALDI_ASSERT(state_ == kHaveScpLine || state_ == kEof); - } - return true; // Success. - } - } - } - - virtual bool IsOpen() const { - switch (state_) { - case kEof: case kError: case kHaveScpLine: case kLoadSucceeded: case kLoadFailed: return true; - case kUninitialized: return false; - default: KALDI_ERR << "IsOpen() called on invalid object."; // kFileStart is not valid - // state for user to call something on. - return false; - } - } - - virtual bool Done() const { - switch (state_) { - case kHaveScpLine: return false; - case kLoadSucceeded: case kLoadFailed: return false; - // These cases are because we want LoadCurrent() - // to be callable after Next() and to not change the Done() status [only Next() should change - // the Done() status]. - case kEof: case kError: return true; // Error condition, like Eof, counts as Done(); the destructor - // or Close() will inform the user of the error. - default: KALDI_ERR << "Done() called on TableReader object at the wrong time."; - return false; - } - } - - virtual std::string Key() { - // Valid to call this whenever Done() returns false. - switch (state_) { - case kHaveScpLine: case kLoadSucceeded: case kLoadFailed: break; - default: - // coding error. - KALDI_ERR << "Key() called on TableReader object at the wrong time."; - } - return key_; - } - const T &Value() { - StateType orig_state = state_; - if (state_ == kHaveScpLine) LoadCurrent(); // Takes - // state_ to kLoadSucceeded or kLoadFailed. - if (state_ == kLoadFailed) { // this can happen due to - // a file listed in an scp file not existing, or - // read failure, failure of a command, etc. - if (orig_state == kHaveScpLine) - KALDI_ERR << "TableReader: failed to load object from " - << PrintableRxfilename(data_rxfilename_) - << " (to suppress this error, add the permissive " - << "(p, ) option to the rspecifier."; - - else // orig_state_ was kLoadFailed, which only could have happened - // if the user called FreeCurrent(). - KALDI_ERR << "TableReader: you called Value() after FreeCurrent()."; - } else if (state_ != kLoadSucceeded) { - // This would be a coding error. - KALDI_ERR << "TableReader: Value() called at the wrong time."; - } - return holder_.Value(); - } - void FreeCurrent() { - if (state_ == kLoadSucceeded) { - holder_.Clear(); - state_ = kLoadFailed; - } else { - KALDI_WARN << "TableReader: FreeCurrent called at the wrong time."; - } - } - void Next() { - while (1) { - NextScpLine(); - if (Done()) return; - if (opts_.permissive) { - // Permissive mode means, when reading scp files, we treat keys whose scp entry - // cannot be read as nonexistent. This means trying to read. - if (LoadCurrent()) return; // Success. - // else try the next scp line. - } else { - return; // We go the next key; Value() will crash if we can't - // read the scp line. - } - } - } - - virtual bool Close() { - // Close() will succeed if the stream was not in an error - // state. To clean up, it also closes the Input objects if - // they're open. - if (script_input_.IsOpen()) - script_input_.Close(); - if (data_input_.IsOpen()) - data_input_.Close(); - if (state_ == kLoadSucceeded) - holder_.Clear(); - if (!this->IsOpen()) - KALDI_ERR << "Close() called on input that was not open."; - StateType old_state = state_; - state_ = kUninitialized; - if (old_state == kError) { - if (opts_.permissive) { - KALDI_WARN << "Close() called on scp file with read error, ignoring the " - "error because permissive mode specified."; - return true; - } else return false; // User will do something with the error status. - } else return true; - } - - virtual ~SequentialTableReaderScriptImpl() { - if (state_ == kError) - KALDI_ERR << "TableReader: reading script file failed: from scp " - << PrintableRxfilename(script_rxfilename_); - // If you don't want this exception to be thrown you can - // call Close() and check the status. - if (state_ == kLoadSucceeded) - holder_.Clear(); - } - private: - bool LoadCurrent() { - // Attempts to load object whose rxfilename is on the current scp line. - if (state_ != kHaveScpLine) - KALDI_ERR << "TableReader: LoadCurrent() called at the wrong time."; - bool ans; - // note, NULL means it doesn't read the binary-mode header - if (Holder::IsReadInBinary()) ans = data_input_.Open(data_rxfilename_, NULL); - else ans = data_input_.OpenTextMode(data_rxfilename_); - if (!ans) { - // May want to make this warning a VLOG at some point - KALDI_WARN << "TableReader: failed to open file " - << PrintableRxfilename(data_rxfilename_); - state_ = kLoadFailed; - return false; - } else { - if (holder_.Read(data_input_.Stream())) { - state_ = kLoadSucceeded; - return true; - } else { // holder_ will not contain data. - KALDI_WARN << "TableReader: failed to load object from " - << PrintableRxfilename(data_rxfilename_); - state_ = kLoadFailed; - return false; - } - } - } - - // Reads the next line in the script file. - void NextScpLine() { - switch (state_) { - case kLoadSucceeded: holder_.Clear(); break; - case kHaveScpLine: case kLoadFailed: case kFileStart: break; - default: - // No other states are valid to call Next() from. - KALDI_ERR << "Reading script file: Next called wrongly."; - } - std::string line; - if (getline(script_input_.Stream(), line)) { - SplitStringOnFirstSpace(line, &key_, &data_rxfilename_); - if (!key_.empty() && !data_rxfilename_.empty()) { - // Got a valid line. - state_ = kHaveScpLine; - } else { - // Got an invalid line. - state_ = kError; // we can't make sense of this - // scp file and will now die. - } - } else { - state_ = kEof; // nothing more in the scp file. - // Might as well close the input streams as don't need them. - script_input_.Close(); - if (data_input_.IsOpen()) - data_input_.Close(); - } - } - - - Input script_input_; // Input object for the .scp file - Input data_input_; // Input object for the entries in - // the script file. - Holder holder_; // Holds the object. - bool binary_; // Binary-mode archive. - std::string key_; - std::string rspecifier_; - std::string script_rxfilename_; // of the script file. - RspecifierOptions opts_; // options. - std::string data_rxfilename_; // of the file we're reading. - enum StateType { - // [The state of the reading process] [does holder_ [is script_inp_ - // have object] open] - kUninitialized, // Uninitialized or closed. no no - kEof, // We did Next() and found eof in script file. no no - kError, // Some other error no yes - kHaveScpLine, // Just called Open() or Next() and have a no yes - // line of the script file but no data. - kLoadSucceeded, // Called LoadCurrent() and it succeeded. yes yes - kLoadFailed, // Called LoadCurrent() and it failed, no yes - // or the user called FreeCurrent().. note, - // if when called by user we are in this state, - // it means the user called FreeCurrent(). - kFileStart, // [state we only use internally] no yes - } state_; - private: -}; - - -// This is the implementation for SequentialTableReader -// when it's an archive. Note that the archive format is: -// key1 [space] object1 key2 [space] -// object2 ... eof. -// "object1" is the output of the Holder::Write function and will -// typically contain a binary header (in binary mode) and then -// the output of object.Write(os, binary). -// The archive itself does not care whether it is in binary -// or text mode, for reading purposes. - -template<class Holder> class SequentialTableReaderArchiveImpl: - public SequentialTableReaderImplBase<Holder> { - public: - typedef typename Holder::T T; - - SequentialTableReaderArchiveImpl(): state_(kUninitialized) { } - - virtual bool Open(const std::string &rspecifier) { - if (state_ != kUninitialized) { - if (! Close()) { // call Close() yourself to suppress this exception. - if (opts_.permissive) - KALDI_WARN << "TableReader::Open, error closing previous input " - "(only warning, since permissive mode)."; - else - KALDI_ERR << "TableReader::Open, error closing previous input."; - } - } - rspecifier_ = rspecifier; - RspecifierType rs = ClassifyRspecifier(rspecifier, - &archive_rxfilename_, - &opts_); - KALDI_ASSERT(rs == kArchiveRspecifier); - - bool ans; - // NULL means don't expect binary-mode header - if (Holder::IsReadInBinary()) - ans = input_.Open(archive_rxfilename_, NULL); - else - ans = input_.OpenTextMode(archive_rxfilename_); - if (!ans) { // header. - KALDI_WARN << "TableReader: failed to open stream " - << PrintableRxfilename(archive_rxfilename_); - state_ = kUninitialized; // Failure on Open - return false; // User should print the error message. - } - state_ = kFileStart; - Next(); - if (state_ == kError) { - KALDI_WARN << "Error beginning to read archive file (wrong filename?): " - << PrintableRxfilename(archive_rxfilename_); - input_.Close(); - state_ = kUninitialized; - return false; - } - KALDI_ASSERT(state_ == kHaveObject || state_ == kEof); - return true; - } - - virtual void Next() { - switch (state_) { - case kHaveObject: - holder_.Clear(); break; - case kFileStart: case kFreedObject: - break; - default: - KALDI_ERR << "TableReader: Next() called wrongly."; - } - std::istream &is = input_.Stream(); - is.clear(); // Clear any fail bits that may have been set... just in case - // this happened in the Read function. - is >> key_; // This eats up any leading whitespace and gets the string. - if (is.eof()) { - state_ = kEof; - return; - } - if (is.fail()) { // This shouldn't really happen, barring file-system errors. - KALDI_WARN << "Error reading archive " - << PrintableRxfilename(archive_rxfilename_); - state_ = kError; - return; - } - int c; - if ((c = is.peek()) != ' ' && c != '\t' && c != '\n') { // We expect a space ' ' after the key. - // We also allow tab [which is consumed] and newline [which is not], just - // so we can read archives generated by scripts that may not be fully - // aware of how this format works. - KALDI_WARN << "Invalid archive file format: expected space after key " - << key_ << ", got character " - << CharToString(static_cast<char>(is.peek())) << ", reading " - << PrintableRxfilename(archive_rxfilename_); - state_ = kError; - return; - } - if (c != '\n') is.get(); // Consume the space or tab. - if (holder_.Read(is)) { - state_ = kHaveObject; - return; - } else { - KALDI_WARN << "Object read failed, reading archive " - << PrintableRxfilename(archive_rxfilename_); - state_ = kError; - return; - } - } - - virtual bool IsOpen() const { - switch (state_) { - case kEof: case kError: case kHaveObject: case kFreedObject: return true; - case kUninitialized: return false; - default: KALDI_ERR << "IsOpen() called on invalid object."; // kFileStart is not valid - // state for user to call something on. - return false; - } - } - - virtual bool Done() const { - switch (state_) { - case kHaveObject: - return false; - case kEof: case kError: - return true; // Error-state counts as Done(), but destructor - // will fail (unless you check the status with Close()). - default: - KALDI_ERR << "Done() called on TableReader object at the wrong time."; - return false; - } - } - - virtual std::string Key() { - // Valid to call this whenever Done() returns false - switch (state_) { - case kHaveObject: break; // only valid case. - default: - // coding error. - KALDI_ERR << "Key() called on TableReader object at the wrong time."; - } - return key_; - } - const T &Value() { - switch (state_) { - case kHaveObject: - break; // only valid case. - default: - // coding error. - KALDI_ERR << "Value() called on TableReader object at the wrong time."; - } - return holder_.Value(); - } - virtual void FreeCurrent() { - if (state_ == kHaveObject) { - holder_.Clear(); - state_ = kFreedObject; - } else - KALDI_WARN << "TableReader: FreeCurernt called at the wrong time."; - } - - virtual bool Close() { - if (! this->IsOpen()) - KALDI_ERR << "Close() called on TableReader twice or otherwise wrongly."; - if (input_.IsOpen()) - input_.Close(); - if (state_ == kHaveObject) - holder_.Clear(); - bool ans; - if (opts_.permissive) { - ans = true; // always return success. - if (state_ == kError) - KALDI_WARN << "Error detected closing TableReader for archive " - << PrintableRxfilename(archive_rxfilename_) << " but ignoring " - << "it as permissive mode specified."; - } else - ans = (state_ != kError); // If error state, user should detect it. - state_ = kUninitialized; - return ans; - } - - virtual ~SequentialTableReaderArchiveImpl() { - if (state_ == kError) { - if (opts_.permissive) - KALDI_WARN << "Error detected closing TableReader for archive " - << PrintableRxfilename(archive_rxfilename_) << " but ignoring " - << "it as permissive mode specified."; - else - KALDI_ERR << "TableReader: error detected closing archive " - << PrintableRxfilename(archive_rxfilename_); - } - // If you don't want this exception to be thrown you can - // call Close() and check the status. - if (state_ == kHaveObject) - holder_.Clear(); - } - private: - Input input_; // Input object for the archive - Holder holder_; // Holds the object. - std::string key_; - std::string rspecifier_; - std::string archive_rxfilename_; - RspecifierOptions opts_; - enum { // [The state of the reading process] [does holder_ [is input_ - // have object] open] - kUninitialized, // Uninitialized or closed. no no - kFileStart, // [state we use internally: just opened.] no yes - kEof, // We did Next() and found eof in archive no no - kError, // Some other error no no - kHaveObject, // We read the key and the object after it. yes yes - kFreedObject, // The user called FreeCurrent(). no yes - } state_; -}; - - -template<class Holder> -SequentialTableReader<Holder>::SequentialTableReader(const std::string &rspecifier): impl_(NULL) { - if (rspecifier != "" && !Open(rspecifier)) - KALDI_ERR << "Error constructing TableReader: rspecifier is " << rspecifier; -} - -template<class Holder> -bool SequentialTableReader<Holder>::Open(const std::string &rspecifier) { - if (IsOpen()) - if (!Close()) - KALDI_ERR << "Could not close previously open object."; - // now impl_ will be NULL. - - RspecifierType wt = ClassifyRspecifier(rspecifier, NULL, NULL); - switch (wt) { - case kArchiveRspecifier: - impl_ = new SequentialTableReaderArchiveImpl<Holder>(); - break; - case kScriptRspecifier: - impl_ = new SequentialTableReaderScriptImpl<Holder>(); - break; - case kNoRspecifier: default: - KALDI_WARN << "Invalid rspecifier " << rspecifier; - return false; - } - if (!impl_->Open(rspecifier)) { - delete impl_; - impl_ = NULL; - return false; // sub-object will have printed warnings. - } - else return true; -} - -template<class Holder> -bool SequentialTableReader<Holder>::Close() { - CheckImpl(); - bool ans = impl_->Close(); - delete impl_; // We don't keep around empty impl_ objects. - impl_ = NULL; - return ans; -} - - -template<class Holder> -bool SequentialTableReader<Holder>::IsOpen() const { - return (impl_ != NULL); // Because we delete the object whenever - // that object is not open. Thus, the IsOpen functions of the - // Impl objects are not really needed. -} - -template<class Holder> -std::string SequentialTableReader<Holder>::Key() { - CheckImpl(); - return impl_->Key(); // this call may throw if called wrongly in other ways, - // e.g. eof. -} - - -template<class Holder> -void SequentialTableReader<Holder>::FreeCurrent() { - CheckImpl(); - impl_->FreeCurrent(); -} - - -template<class Holder> -const typename SequentialTableReader<Holder>::T & -SequentialTableReader<Holder>::Value() { - CheckImpl(); - return impl_->Value(); // This may throw (if LoadCurrent() returned false you are safe.). -} - - -template<class Holder> -void SequentialTableReader<Holder>::Next() { - CheckImpl(); - impl_->Next(); -} - -template<class Holder> -bool SequentialTableReader<Holder>::Done() { - CheckImpl(); - return impl_->Done(); -} - - -template<class Holder> -SequentialTableReader<Holder>::~SequentialTableReader() { - if (impl_) delete impl_; - // Destructor of impl_ may throw. -} - - - -template<class Holder> class TableWriterImplBase { - public: - typedef typename Holder::T T; - - virtual bool Open(const std::string &wspecifier) = 0; - - // Write returns true on success, false on failure, but - // some errors may not be detected until we call Close(). - // It throws (via KALDI_ERR) if called wrongly. We could - // have just thrown on all errors, since this is what - // TableWriter does; it was designed this way because originally - // TableWriter::Write returned an exit status. - virtual bool Write(const std::string &key, const T &value) = 0; - - // Flush will flush any archive; it does not return error status, - // any errors will be reported on the next Write or Close. - virtual void Flush() = 0; - - virtual bool Close() = 0; - - virtual bool IsOpen() const = 0; - - // May throw on write error if Close was not called. - virtual ~TableWriterImplBase() { } - - TableWriterImplBase() { } - private: - KALDI_DISALLOW_COPY_AND_ASSIGN(TableWriterImplBase); -}; - - -// The implementation of TableWriter we use when writing directly -// to an archive with no associated scp. -template<class Holder> -class TableWriterArchiveImpl: public TableWriterImplBase<Holder> { - public: - typedef typename Holder::T T; - - virtual bool Open(const std::string &wspecifier) { - switch (state_) { - case kUninitialized: - break; - case kWriteError: - KALDI_ERR << "TableWriter: opening stream, already open with write error."; - case kOpen: default: - if (!Close()) // throw because this error may not have been previously - // detected by the user. - KALDI_ERR << "TableWriter: opening stream, error closing previously open stream."; - } - wspecifier_ = wspecifier; - WspecifierType ws = ClassifyWspecifier(wspecifier, - &archive_wxfilename_, - NULL, - &opts_); - KALDI_ASSERT(ws == kArchiveWspecifier); // or wrongly called. - - if (output_.Open(archive_wxfilename_, opts_.binary, false)) { // false means no binary header. - state_ = kOpen; - return true; - } else { - // stream will not be open. User will report this error - // (we return bool), so don't bother printing anything. - state_ = kUninitialized; - return false; - } - } - - virtual bool IsOpen() const { - switch (state_) { - case kUninitialized: return false; - case kOpen: case kWriteError: return true; - default: KALDI_ERR << "IsOpen() called on TableWriter in invalid state."; - } - return false; - } - - // Write returns true on success, false on failure, but - // some errors may not be detected till we call Close(). - virtual bool Write(const std::string &key, const T &value) { - switch (state_) { - case kOpen: break; - case kWriteError: - // user should have known from the last - // call to Write that there was a problem. - KALDI_WARN << "TableWriter: attempting to write to invalid stream."; - return false; - case kUninitialized: default: - KALDI_ERR << "TableWriter: Write called on invalid stream"; - - } - // state is now kOpen or kWriteError. - if (!IsToken(key)) // e.g. empty string or has spaces... - KALDI_ERR << "TableWriter: using invalid key " << key; - output_.Stream() << key << ' '; - if (!Holder::Write(output_.Stream(), opts_.binary, value)) { - KALDI_WARN << "TableWriter: write failure to " - << PrintableWxfilename(archive_wxfilename_); - state_ = kWriteError; - return false; - } - if (state_ == kWriteError) return false; // Even if this Write seems to have - // succeeded, we fail because a previous Write failed and the archive may be - // corrupted and unreadable. - - if (opts_.flush) - Flush(); - return true; - } - - // Flush will flush any archive; it does not return error status, - // any errors will be reported on the next Write or Close. - virtual void Flush() { - switch (state_) { - case kWriteError: case kOpen: - output_.Stream().flush(); // Don't check error status. - return; - default: - KALDI_WARN << "TableWriter: Flush called on not-open writer."; - } - } - - virtual bool Close() { - if (!this->IsOpen() || !output_.IsOpen()) - KALDI_ERR << "TableWriter: Close called on a stream that was not open." << this->IsOpen() << ", " << output_.IsOpen(); - bool close_success = output_.Close(); - if (!close_success) { - KALDI_WARN << "TableWriter: error closing stream: wspecifier is " - << wspecifier_; - state_ = kUninitialized; - return false; - } - if (state_ == kWriteError) { - KALDI_WARN << "TableWriter: closing writer in error state: wspecifier is " - << wspecifier_; - state_ = kUninitialized; - return false; - } - state_ = kUninitialized; - return true; - } - - TableWriterArchiveImpl(): state_(kUninitialized) {} - - // May throw on write error if Close was not called. - virtual ~TableWriterArchiveImpl() { - if (!IsOpen()) return; - else if (!Close()) - KALDI_ERR << "At TableWriter destructor: Write failed or stream close " - << "failed: wspecifier is "<< wspecifier_; - } - - private: - Output output_; - WspecifierOptions opts_; - std::string wspecifier_; - std::string archive_wxfilename_; - enum { // is stream open? - kUninitialized, // no - kOpen, // yes - kWriteError, // yes - } state_; -}; - - - - -// The implementation of TableWriter we use when writing to -// individual files (more generally, wxfilenames) specified -// in an scp file that we read. - -// Note: the code for this class is similar to RandomAccessTableReaderScriptImpl; -// try to keep them in sync. - -template<class Holder> -class TableWriterScriptImpl: public TableWriterImplBase<Holder> { - public: - typedef typename Holder::T T; - - TableWriterScriptImpl(): last_found_(0), state_(kUninitialized) {} - - virtual bool Open(const std::string &wspecifier) { - switch (state_) { - case kReadScript: - KALDI_ERR << " Opening already open TableWriter: call Close first."; - case kUninitialized: case kNotReadScript: - break; - } - wspecifier_ = wspecifier; - WspecifierType ws = ClassifyWspecifier(wspecifier, - NULL, - &script_rxfilename_, - &opts_); - KALDI_ASSERT(ws == kScriptWspecifier); // or wrongly called. - KALDI_ASSERT(script_.empty()); // no way it could be nonempty at this point. - - if (! ReadScriptFile(script_rxfilename_, - true, // print any warnings - &script_)) { // error reading script file or invalid format - state_ = kNotReadScript; - return false; // no need to print further warnings. user gets the error. - } - std::sort(script_.begin(), script_.end()); - for (size_t i = 0; i+1 < script_.size(); i++) { - if (script_[i].first.compare(script_[i+1].first) >= 0) { - // script[i] not < script[i+1] in lexical order... - KALDI_WARN << "Script file " << PrintableRxfilename(script_rxfilename_) - << " contains duplicate key " << script_[i].first; - state_ = kNotReadScript; - return false; - } - } - state_ = kReadScript; - return true; - } - - virtual bool IsOpen() const { return (state_ == kReadScript); } - - virtual bool Close() { - if (!IsOpen()) - KALDI_ERR << "Close() called on TableWriter that was not open."; - state_ = kUninitialized; - last_found_ = 0; - script_.clear(); - return true; - } - - // Write returns true on success, false on failure, but - // some errors may not be detected till we call Close(). - virtual bool Write(const std::string &key, const T &value) { - if (!IsOpen()) - KALDI_ERR << "TableWriter: Write called on invalid stream"; - - if (!IsToken(key)) // e.g. empty string or has spaces... - KALDI_ERR << "TableWriter: using invalid key " << key; - - std::string wxfilename; - if (!LookupFilename(key, &wxfilename)) { - if (opts_.permissive) { - return true; // In permissive mode, it's as if we're writing to /dev/null - // for missing keys. - } else { - KALDI_WARN << "TableWriter: script file " - << PrintableRxfilename(script_rxfilename_) - << " has no entry for key "<<key; - return false; - } - } - Output output; - if (!output.Open(wxfilename, opts_.binary, false)) { - // Open in the text/binary mode (on Windows) given by member var. "binary" - // (obtained from wspecifier), but do not put the binary-mode header (it - // will be written, if needed, by the Holder::Write function.) - KALDI_WARN << "TableWriter: failed to open stream: " - << PrintableWxfilename(wxfilename); - return false; - } - if (!Holder::Write(output.Stream(), opts_.binary, value) - || !output.Close()) { - KALDI_WARN << "TableWriter: failed to write data to " - << PrintableWxfilename(wxfilename); - return false; - } - return true; - } - - // Flush does nothing in this implementation, there is nothing to flush. - virtual void Flush() { } - - - virtual ~TableWriterScriptImpl() { - // Nothing to do in destructor. - } - - private: - // Note: this function is almost the same as in RandomAccessTableReaderScriptImpl. - bool LookupFilename(const std::string &key, std::string *wxfilename) { - // First, an optimization: if we're going consecutively, this will - // make the lookup very fast. - last_found_++; - if (last_found_ < script_.size() && script_[last_found_].first == key) { - *wxfilename = script_[last_found_].second; - return true; - } - std::pair<std::string, std::string> pr(key, ""); // Important that "" - // compares less than or equal to any string, so lower_bound points to the - // element that has the same key. - typedef typename std::vector<std::pair<std::string, std::string> >::const_iterator - IterType; - IterType iter = std::lower_bound(script_.begin(), script_.end(), pr); - if (iter != script_.end() && iter->first == key) { - last_found_ = iter - script_.begin(); - *wxfilename = iter->second; - return true; - } else { - return false; - } - } - - - WspecifierOptions opts_; - std::string wspecifier_; - std::string script_rxfilename_; - - // the script_ variable contains pairs of (key, filename), sorted using - // std::sort. This can be used with binary_search to look up filenames for - // writing. If this becomes inefficient we can use std::unordered_map (but I - // suspect this wouldn't be significantly faster & would use more memory). - // If memory becomes a problem here, the user should probably be passing - // only the relevant part of the scp file rather than expecting us to get too - // clever in the code. - std::vector<std::pair<std::string, std::string> > script_; - size_t last_found_; // This is for an optimization used in LookupFilename. - - enum { - kUninitialized, - kReadScript, - kNotReadScript, // read of script failed. - } state_; -}; - - -// The implementation of TableWriter we use when writing directly -// to an archive plus an associated scp. -template<class Holder> -class TableWriterBothImpl: public TableWriterImplBase<Holder> { - public: - typedef typename Holder::T T; - - virtual bool Open(const std::string &wspecifier) { - switch (state_) { - case kUninitialized: - break; - case kWriteError: - KALDI_ERR << "TableWriter: opening stream, already open with write error."; - case kOpen: default: - if (!Close()) // throw because this error may not have been previously detected by user. - KALDI_ERR << "TableWriter: opening stream, error closing previously open stream."; - } - wspecifier_ = wspecifier; - WspecifierType ws = ClassifyWspecifier(wspecifier, - &archive_wxfilename_, - &script_wxfilename_, - &opts_); - KALDI_ASSERT(ws == kBothWspecifier); // or wrongly called. - if (ClassifyWxfilename(archive_wxfilename_) != kFileOutput) - KALDI_WARN << "When writing to both archive and script, the script file " - "will generally not be interpreted correctly unless the archive is " - "an actual file: wspecifier = " << wspecifier; - - if (!archive_output_.Open(archive_wxfilename_, opts_.binary, false)) { // false means no binary header. - state_ = kUninitialized; - return false; - } - if (!script_output_.Open(script_wxfilename_, false, false)) { // first false means text mode: - // script files always text-mode. second false means don't write header (doesn't matter - // for text mode). - archive_output_.Close(); // Don't care about status: error anyway. - state_ = kUninitialized; - return false; - } - state_ = kOpen; - return true; - } - - virtual bool IsOpen() const { - switch (state_) { - case kUninitialized: return false; - case kOpen: case kWriteError: return true; - default: KALDI_ERR << "IsOpen() called on TableWriter in invalid state."; - } - return false; - } - - void MakeFilename(typename std::ostream::pos_type streampos, std::string *output) const { - std::ostringstream ss; - ss << ':' << streampos; - KALDI_ASSERT(ss.str() != ":-1"); - *output = archive_wxfilename_ + ss.str(); - - // e.g. /some/file:12302. - // Note that we warned if archive_wxfilename_ is not an actual filename; - // the philosophy is we give the user rope and if they want to hang - // themselves, with it, fine. - } - - // Write returns true on success, false on failure, but - // some errors may not be detected till we call Close(). - virtual bool Write(const std::string &key, const T &value) { - switch (state_) { - case kOpen: break; - case kWriteError: - // user should have known from the last - // call to Write that there was a problem. Warn about it. - KALDI_WARN << "TableWriter: writing to non-open TableWriter object."; - return false; - case kUninitialized: default: - KALDI_ERR << "TableWriter: Write called on invalid stream"; - } - // state is now kOpen or kWriteError. - if (!IsToken(key)) // e.g. empty string or has spaces... - KALDI_ERR << "TableWriter: using invalid key " << key; - std::ostream &archive_os = archive_output_.Stream(); - archive_os << key << ' '; - typename std::ostream::pos_type archive_os_pos = archive_os.tellp(); - // position at start of Write() to archive. We will record this in the script file. - std::string offset_rxfilename; // rxfilename with offset into the archive, - // e.g. some_archive_name.ark:431541423 - MakeFilename(archive_os_pos, &offset_rxfilename); - - // Write to the script file first. - // The idea is that we want to get all the information possible into the - // script file, to make it easier to unwind errors later. - std::ostream &script_os = script_output_.Stream(); - script_output_.Stream() << key << ' ' << offset_rxfilename << '\n'; - - if (!Holder::Write(archive_output_.Stream(), opts_.binary, value)) { - KALDI_WARN << "TableWriter: write failure to" - << PrintableWxfilename(archive_wxfilename_); - state_ = kWriteError; - return false; - } - - if (script_os.fail()) { - KALDI_WARN << "TableWriter: write failure to script file detected: " - << PrintableWxfilename(script_wxfilename_); - state_ = kWriteError; - return false; - } - - if (archive_os.fail()) { - KALDI_WARN << "TableWriter: write failure to archive file detected: " - << PrintableWxfilename(archive_wxfilename_); - state_ = kWriteError; - return false; - } - - if (state_ == kWriteError) return false; // Even if this Write seems to have - // succeeded, we fail because a previous Write failed and the archive may be - // corrupted and unreadable. - - if (opts_.flush) - Flush(); - return true; - } - - // Flush will flush any archive; it does not return error status, - // any errors will be reported on the next Write or Close. - virtual void Flush() { - switch (state_) { - case kWriteError: case kOpen: - archive_output_.Stream().flush(); // Don't check error status. - script_output_.Stream().flush(); // Don't check error status. - return; - default: - KALDI_WARN << "TableWriter: Flush called on not-open writer."; - } - } - - virtual bool Close() { - if (!this->IsOpen()) - KALDI_ERR << "TableWriter: Close called on a stream that was not open."; - bool close_success = true; - if (archive_output_.IsOpen()) - if (!archive_output_.Close()) close_success = false; - if (script_output_.IsOpen()) - if (!script_output_.Close()) close_success = false; - bool ans = close_success && (state_ != kWriteError); - state_ = kUninitialized; - return ans; - } - - TableWriterBothImpl(): state_(kUninitialized) {} - - // May throw on write error if Close() was not called. - // User can get the error status by calling Close(). - virtual ~TableWriterBothImpl() { - if (!IsOpen()) return; - else if (!Close()) - KALDI_ERR << "At TableWriter destructor: Write failed or stream close failed: " - << wspecifier_; - } - - private: - Output archive_output_; - Output script_output_; - WspecifierOptions opts_; - std::string archive_wxfilename_; - std::string script_wxfilename_; - std::string wspecifier_; - enum { // is stream open? - kUninitialized, // no - kOpen, // yes - kWriteError, // yes - } state_; -}; - - -template<class Holder> -TableWriter<Holder>::TableWriter(const std::string &wspecifier): impl_(NULL) { - if (wspecifier != "" && !Open(wspecifier)) { - KALDI_ERR << "TableWriter: failed to write to " - << wspecifier; - } -} - -template<class Holder> -bool TableWriter<Holder>::IsOpen() const { - return (impl_ != NULL); -} - - -template<class Holder> -bool TableWriter<Holder>::Open(const std::string &wspecifier) { - - if (IsOpen()) { - if (!Close()) // call Close() yourself to suppress this exception. - KALDI_ERR << "TableWriter::Open, failed to close previously open writer."; - } - KALDI_ASSERT(impl_ == NULL); - WspecifierType wtype = ClassifyWspecifier(wspecifier, NULL, NULL, NULL); - switch (wtype) { - case kBothWspecifier: - impl_ = new TableWriterBothImpl<Holder>(); - break; - case kArchiveWspecifier: - impl_ = new TableWriterArchiveImpl<Holder>(); - break; - case kScriptWspecifier: - impl_ = new TableWriterScriptImpl<Holder>(); - break; - case kNoWspecifier: default: - KALDI_WARN << "ClassifyWspecifier: invalid wspecifier " << wspecifier; - return false; - } - if (impl_->Open(wspecifier)) return true; - else { // The class will have printed a more specific warning. - delete impl_; - impl_ = NULL; - return false; - } -} - -template<class Holder> -void TableWriter<Holder>::Write(const std::string &key, - const T &value) const { - CheckImpl(); - if (!impl_->Write(key, value)) - KALDI_ERR << "Error in TableWriter::Write"; - // More specific warning will have - // been printed in the Write function. -} - -template<class Holder> -void TableWriter<Holder>::Flush() { - CheckImpl(); - impl_->Flush(); -} - -template<class Holder> -bool TableWriter<Holder>::Close() { - CheckImpl(); - bool ans = impl_->Close(); - delete impl_; // We don't keep around non-open impl_ objects [c.f. definition of IsOpen()] - impl_ = NULL; - return ans; -} - -template<class Holder> -TableWriter<Holder>::~TableWriter() { - if (IsOpen() && !Close()) { - KALDI_ERR << "Error closing TableWriter [in destructor]."; - } -} - - -// Types of RandomAccessTableReader: -// In principle, we would like to have four types of RandomAccessTableReader: -// the 4 combinations [scp, archive], [seekable, not-seekable], -// where if something is seekable we only store a file offset. However, -// it seems sufficient for now to only implement two of these, in both -// cases assuming it's not seekable so we never store file offsets and always -// store either the scp line or the data in the archive. The reasons are: -// (1) -// For scp files, storing the actual entry is not that much more expensive -// than storing the file offsets (since the entries are just filenames), and -// avoids a lot of fseek operations that might be expensive. -// (2) -// For archive files, there is no real reason, if you have the archive file -// on disk somewhere, why you wouldn't access it via its associated scp. -// [i.e. write it as ark, scp]. The main reason to read archives directly -// is if they are part of a pipe, and in this case it's not seekable, so -// we implement only this case. -// -// Note that we will rarely in practice have to keep in memory everything in -// the archive, as long as things are only read once from the archive (the -// "o, " or "once" option) and as long as we keep our keys in sorted order; to take -// advantage of this we need the "s, " (sorted) option, so we would read archives -// as e.g. "s, o, ark:-" (this is the rspecifier we would use if it was the -// standard input and these conditions held). - -template<class Holder> class RandomAccessTableReaderImplBase { - public: - typedef typename Holder::T T; - - virtual bool Open(const std::string &rspecifier) = 0; - - virtual bool HasKey(const std::string &key) = 0; - - virtual const T &Value(const std::string &key) = 0; - - virtual bool Close() = 0; - - virtual ~RandomAccessTableReaderImplBase() {} -}; - - -// Implementation of RandomAccessTableReader for a script file; for simplicity we -// just read it in all in one go, as it's unlikely someone would generate this -// from a pipe. In principle we could read it on-demand as for the archives, but -// this would probably be overkill. - -// Note: the code for this this class is similar to TableWriterScriptImpl: -// try to keep them in sync. -template<class Holder> -class RandomAccessTableReaderScriptImpl: - public RandomAccessTableReaderImplBase<Holder> { - - public: - typedef typename Holder::T T; - - RandomAccessTableReaderScriptImpl(): last_found_(0), state_(kUninitialized) {} - - virtual bool Open(const std::string &rspecifier) { - switch (state_) { - case kNotHaveObject: case kHaveObject: case kGaveObject: - KALDI_ERR << " Opening already open RandomAccessTableReader: call Close first."; - case kUninitialized: case kNotReadScript: - break; - } - rspecifier_ = rspecifier; - RspecifierType rs = ClassifyRspecifier(rspecifier, - &script_rxfilename_, - &opts_); - KALDI_ASSERT(rs == kScriptRspecifier); // or wrongly called. - KALDI_ASSERT(script_.empty()); // no way it could be nonempty at this point. - - if (! ReadScriptFile(script_rxfilename_, - true, // print any warnings - &script_)) { // error reading script file or invalid format - state_ = kNotReadScript; - return false; // no need to print further warnings. user gets the error. - } - - rspecifier_ = rspecifier; - // If opts_.sorted, the user has asserted that the keys are already sorted. - // Although we could easily sort them, we want to let the user know of this - // mistake. This same mistake could have serious effects if used with an - // archive rather than a script. - if (!opts_.sorted) - std::sort(script_.begin(), script_.end()); - for (size_t i = 0; i+1 < script_.size(); i++) { - if (script_[i].first.compare(script_[i+1].first) >= 0) { - // script[i] not < script[i+1] in lexical order... - bool same = (script_[i].first == script_[i+1].first); - KALDI_WARN << "Script file " << PrintableRxfilename(script_rxfilename_) - << (same ? " contains duplicate key: " : - " is not sorted (remove s, option or add ns, option): key is ") - << script_[i].first; - state_ = kNotReadScript; - return false; - } - } - state_ = kNotHaveObject; - return true; - } - - virtual bool IsOpen() const { - return (state_ == kNotHaveObject || state_ == kHaveObject || - state_ == kGaveObject); - } - - virtual bool Close() { - if (!IsOpen()) - KALDI_ERR << "Close() called on RandomAccessTableReader that was not open."; - holder_.Clear(); - state_ = kUninitialized; - last_found_ = 0; - script_.clear(); - current_key_ = ""; - // This one cannot fail because any errors of a "global" - // nature would have been detected when we did Open(). - // With archives it's different. - return true; - } - - virtual bool HasKey(const std::string &key) { - bool preload = opts_.permissive; - // In permissive mode, we have to check that we can read - // the scp entry before we assert that the key is there. - return HasKeyInternal(key, preload); - } - - - // Write returns true on success, false on failure, but - // some errors may not be detected till we call Close(). - virtual const T& Value(const std::string &key) { - - if (!IsOpen()) - KALDI_ERR << "Value() called on non-open object."; - - if (!((state_ == kHaveObject || state_ == kGaveObject) - && key == current_key_)) { // Not already stored... - bool has_key = HasKeyInternal(key, true); // preload. - if (!has_key) - KALDI_ERR << "Could not get item for key " << key - << ", rspecifier is " << rspecifier_ << "[to ignore this, " - << "add the p, (permissive) option to the rspecifier."; - KALDI_ASSERT(state_ == kHaveObject && key == current_key_); - } - - if (state_ == kHaveObject) { - state_ = kGaveObject; - if (opts_.once) MakeTombstone(key); // make sure that future lookups fail. - return holder_.Value(); - } else { // state_ == kGaveObject - if (opts_.once) - KALDI_ERR << "Value called twice for the same key and ,o (once) option " - << "is used: rspecifier is " << rspecifier_; - return holder_.Value(); - } - } - - virtual ~RandomAccessTableReaderScriptImpl() { - if (state_ == kHaveObject || state_ == kGaveObject) - holder_.Clear(); - } - - private: - // HasKeyInternal when called with preload == false just tells us whether the - // key is in the scp. With preload == true, which happens when the ,p - // (permissive) option is given in the rspecifier, it will also check that we - // can preload the object from disk (loading from the rxfilename in the scp), - // and only return true if we can. This function is called both from HasKey - // and from Value(). - virtual bool HasKeyInternal(const std::string &key, bool preload) { - switch (state_) { - case kUninitialized: case kNotReadScript: - KALDI_ERR << "HasKey called on RandomAccessTableReader object that is not open."; - case kHaveObject: case kGaveObject: - if (key == current_key_) - return true; - break; - default: break; - } - KALDI_ASSERT(IsToken(key)); - size_t key_pos = 0; // set to zero to suppress warning - bool ans = LookupKey(key, &key_pos); - if (!ans) return false; - else { - // First do a check regarding the "once" option. - if (opts_.once && script_[key_pos].second == "") { // A "tombstone"; user is asking about - // already-read key. - KALDI_ERR << "HasKey called on key whose value was already read, and " - " you specified the \"once\" option (o, ): try removing o, or adding no, :" - " rspecifier is " << rspecifier_; - } - if (!preload) - return true; // we have the key. - else { // preload specified, so we have to pre-load the object before returning true. - if (!input_.Open(script_[key_pos].second)) { - KALDI_WARN << "Error opening stream " - << PrintableRxfilename(script_[key_pos].second); - return false; - } else { - // Make sure holder empty. - if (state_ == kHaveObject || state_ == kGaveObject) - holder_.Clear(); - if (holder_.Read(input_.Stream())) { - state_ = kHaveObject; - current_key_ = key; - return true; - } else { - KALDI_WARN << "Error reading object from " - "stream " << PrintableRxfilename(script_[key_pos].second); - state_ = kNotHaveObject; - return false; - } - } - } - } - } - void MakeTombstone(const std::string &key) { - size_t offset; - if (!LookupKey(key, &offset)) - KALDI_ERR << "RandomAccessTableReader object in inconsistent state."; - else - script_[offset].second = ""; - } - bool LookupKey(const std::string &key, size_t *script_offset) { - // First, an optimization: if we're going consecutively, this will - // make the lookup very fast. Since we may call HasKey and then - // Value(), which both may look up the key, we test if either the - // current or next position are correct. - if (last_found_ < script_.size() && script_[last_found_].first == key) { - *script_offset = last_found_; - return true; - } - last_found_++; - if (last_found_ < script_.size() && script_[last_found_].first == key) { - *script_offset = last_found_; - return true; - } - std::pair<std::string, std::string> pr(key, ""); // Important that "" - // compares less than or equal to any string, so lower_bound points to the - // element that has the same key. - typedef typename std::vector<std::pair<std::string, std::string> >::const_iterator - IterType; - IterType iter = std::lower_bound(script_.begin(), script_.end(), pr); - if (iter != script_.end() && iter->first == key) { - last_found_ = *script_offset = iter - script_.begin(); - return true; - } else { - return false; - } - } - - - Input input_; // Use the same input_ object for reading each file, in case - // the scp specifies offsets in an archive (so we can keep the same file open). - RspecifierOptions opts_; - std::string rspecifier_; // rspecifier used to open it; used in debug messages - std::string script_rxfilename_; // filename of script. - - std::string current_key_; // Key of object in holder_ - Holder holder_; - - // the script_ variable contains pairs of (key, filename), sorted using - // std::sort. This can be used with binary_search to look up filenames for - // writing. If this becomes inefficient we can use std::unordered_map (but I - // suspect this wouldn't be significantly faster & would use more memory). - // If memory becomes a problem here, the user should probably be passing - // only the relevant part of the scp file rather than expecting us to get too - // clever in the code. - std::vector<std::pair<std::string, std::string> > script_; - size_t last_found_; // This is for an optimization used in FindFilename. - - enum { // [Do we have [Does holder_ - // script_ set up?] contain object?] - kUninitialized, // no no - kNotReadScript, // no no - kNotHaveObject, // yes no - kHaveObject, // yes yes - kGaveObject, // yes yes - // [kGaveObject is as kHaveObject but we note that the - // user has already read it; this is for checking that - // if "once" is specified, the user actually only reads - // it once. - } state_; - -}; - - - - -// This is the base-class (with some implemented functions) for the -// implementations of RandomAccessTableReader when it's an archive. This -// base-class handles opening the files, storing the state of the reading -// process, and loading objects. This is the only case in which we have -// an intermediate class in the hierarchy between the virtual ImplBase -// class and the actual Impl classes. -// The child classes vary in the assumptions regarding sorting, etc. - -template<class Holder> class RandomAccessTableReaderArchiveImplBase: - public RandomAccessTableReaderImplBase<Holder> { - public: - typedef typename Holder::T T; - - RandomAccessTableReaderArchiveImplBase(): holder_(NULL), state_(kUninitialized) { } - - virtual bool Open(const std::string &rspecifier) { - if (state_ != kUninitialized) { - if (! this->Close()) // call Close() yourself to suppress this exception. - KALDI_ERR << "TableReader::Open, error closing previous input."; - } - rspecifier_ = rspecifier; - RspecifierType rs = ClassifyRspecifier(rspecifier, &archive_rxfilename_, - &opts_); - KALDI_ASSERT(rs == kArchiveRspecifier); - - // NULL means don't expect binary-mode header - bool ans; - if (Holder::IsReadInBinary()) - ans = input_.Open(archive_rxfilename_, NULL); - else - ans = input_.OpenTextMode(archive_rxfilename_); - if (!ans) { // header. - KALDI_WARN << "TableReader: failed to open stream " - << PrintableRxfilename(archive_rxfilename_); - state_ = kUninitialized; // Failure on Open - return false; // User should print the error message. - } else { - state_ = kNoObject; - } - return true; - } - - // ReadNextObject() requires that the state be kNoObject, - // and it will try read the next object. If it succeeds, - // it sets the state to kHaveObject, and - // cur_key_ and holder_ have the key and value. If it fails, - // it sets the state to kError or kEof. - void ReadNextObject() { - if (state_ != kNoObject) - KALDI_ERR << "TableReader: ReadNextObject() called from wrong state."; // Code error - // somewhere in this class or a child class. - std::istream &is = input_.Stream(); - is.clear(); // Clear any fail bits that may have been set... just in case - // this happened in the Read function. - is >> cur_key_; // This eats up any leading whitespace and gets the string. - if (is.eof()) { - state_ = kEof; - return; - } - if (is.fail()) { // This shouldn't really happen, barring file-system errors. - KALDI_WARN << "Error reading archive: rspecifier is " << rspecifier_; - state_ = kError; - return; - } - int c; - if ((c = is.peek()) != ' ' && c != '\t' && c != '\n') { // We expect a space ' ' after the key. - // We also allow tab, just so we can read archives generated by scripts that may - // not be fully aware of how this format works. - KALDI_WARN << "Invalid archive file format: expected space after key " <<cur_key_ - <<", got character " - << CharToString(static_cast<char>(is.peek())) << ", reading archive " - << PrintableRxfilename(archive_rxfilename_); - state_ = kError; - return; - } - if (c != '\n') is.get(); // Consume the space or tab. - holder_ = new Holder; - if (holder_->Read(is)) { - state_ = kHaveObject; - return; - } else { - KALDI_WARN << "Object read failed, reading archive " - << PrintableRxfilename(archive_rxfilename_); - state_ = kError; - delete holder_; - holder_ = NULL; - return; - } - } - - virtual bool IsOpen() const { - switch (state_) { - case kEof: case kError: case kHaveObject: case kNoObject: return true; - case kUninitialized: return false; - default: KALDI_ERR << "IsOpen() called on invalid object."; - return false; - } - } - - // Called by the child-class virutal Close() functions; does the - // shared parts of the cleanup. - bool CloseInternal() { - if (! this->IsOpen()) - KALDI_ERR << "Close() called on TableReader twice or otherwise wrongly."; - if (input_.IsOpen()) - input_.Close(); - if (state_ == kHaveObject) { - KALDI_ASSERT(holder_ != NULL); - delete holder_; - holder_ = NULL; - } else KALDI_ASSERT(holder_ == NULL); - bool ans = (state_ != kError); - state_ = kUninitialized; - if (!ans && opts_.permissive) { - KALDI_WARN << "Error state detected closing reader. " - << "Ignoring it because you specified permissive mode."; - return true; - } - return ans; - } - - ~RandomAccessTableReaderArchiveImplBase() { - // The child class has the responsibility to call CloseInternal(). - KALDI_ASSERT(state_ == kUninitialized && holder_ == NULL); - } - private: - Input input_; // Input object for the archive - protected: - // The variables below are accessed by child classes. - - std::string cur_key_; // current key (if state == kHaveObject). - Holder *holder_; // Holds the object we just read (if state == kHaveObject) - - std::string rspecifier_; - std::string archive_rxfilename_; - RspecifierOptions opts_; - - enum { // [The state of the reading process] [does holder_ [is input_ - // have object] open] - kUninitialized, // Uninitialized or closed no no - kNoObject, // Do not have object in holder_ no yes - kHaveObject, // Have object in holder_ yes yes - kEof, // End of file no yes - kError, // Some kind of error-state in the reading. no yes - } state_; - -}; - - -// RandomAccessTableReaderDSortedArchiveImpl (DSorted for "doubly sorted") is the -// implementation for random-access reading of archives when both the archive, -// and the calling code, are in sorted order (i.e. we ask for the keys in sorted -// order). This is when the s and cs options are both given. It only ever has -// to keep one object in memory. It inherits from -// RandomAccessTableReaderArchiveImplBase which implements the common parts of -// RandomAccessTableReader that are used when it's an archive we're reading from. - -template<class Holder> class RandomAccessTableReaderDSortedArchiveImpl: - public RandomAccessTableReaderArchiveImplBase<Holder> { - using RandomAccessTableReaderArchiveImplBase<Holder>::kUninitialized; - using RandomAccessTableReaderArchiveImplBase<Holder>::kHaveObject; - using RandomAccessTableReaderArchiveImplBase<Holder>::kNoObject; - using RandomAccessTableReaderArchiveImplBase<Holder>::kEof; - using RandomAccessTableReaderArchiveImplBase<Holder>::kError; - using RandomAccessTableReaderArchiveImplBase<Holder>::state_; - using RandomAccessTableReaderArchiveImplBase<Holder>::opts_; - using RandomAccessTableReaderArchiveImplBase<Holder>::cur_key_; - using RandomAccessTableReaderArchiveImplBase<Holder>::holder_; - using RandomAccessTableReaderArchiveImplBase<Holder>::rspecifier_; - using RandomAccessTableReaderArchiveImplBase<Holder>::archive_rxfilename_; - using RandomAccessTableReaderArchiveImplBase<Holder>::ReadNextObject; - public: - typedef typename Holder::T T; - - RandomAccessTableReaderDSortedArchiveImpl() { } - - virtual bool Close() { - // We don't have anything additional to clean up, so just - // call generic base-class one. - return this->CloseInternal(); - } - - virtual bool HasKey(const std::string &key) { - return FindKeyInternal(key); - } - virtual const T & Value(const std::string &key) { - if (FindKeyInternal(key)) { - KALDI_ASSERT(this->state_ == kHaveObject && key == this->cur_key_ - && holder_ != NULL); - return this->holder_->Value(); - } else { - KALDI_ERR << "Value() called but no such key " << key - << " in archive " << PrintableRxfilename(archive_rxfilename_); - return *(const T*)NULL; // keep compiler happy. - } - } - - virtual ~RandomAccessTableReaderDSortedArchiveImpl() { - if (this->IsOpen()) - if (!Close()) // more specific warning will already have been printed. - // we are in some kind of error state & user did not find out by - // calling Close(). - KALDI_ERR << "Error closing RandomAccessTableReader: rspecifier is " - << rspecifier_; - } - private: - // FindKeyInternal tries to find the key by calling "ReadNextObject()" - // as many times as necessary till we get to it. It is called from - // both FindKey and Value(). - bool FindKeyInternal(const std::string &key) { - // First check that the user is calling us right: should be - // in sorted order. If not, error. - if (!last_requested_key_.empty()) { - if (key.compare(last_requested_key_) < 0) { // key < last_requested_key_ - KALDI_ERR << "You provided the \"cs\" option " - << "but are not calling with keys in sorted order: " - << key << " < " << last_requested_key_ << ": rspecifier is " - << rspecifier_; - } - } - // last_requested_key_ is just for debugging of order of calling. - last_requested_key_ = key; - - if (state_ == kNoObject) - ReadNextObject(); // This can only happen - // once, the first time someone calls HasKey() or Value(). We don't - // do it in the initializer to stop the program hanging too soon, - // if reading from a pipe. - - if (state_ == kEof || state_ == kError) return false; - - if (state_ == kUninitialized) - KALDI_ERR << "Trying to access a RandomAccessTableReader object that is not open."; - - std::string last_key_; // To check that - // the archive we're reading is in sorted order. - while (1) { - KALDI_ASSERT(state_ == kHaveObject); - int compare = key.compare(cur_key_); - if (compare == 0) { // key == key_ - return true; // we got it.. - } else if (compare < 0) { // key < cur_key_, so we already read past the - // place where we want to be. This implies that we will never find it - // [due to the sorting etc., this means it just isn't in the archive]. - return false; - } else { // compare > 0, key > cur_key_. We need to read further ahead. - last_key_ = cur_key_; - // read next object.. we have to set state to kNoObject first. - KALDI_ASSERT(holder_ != NULL); - delete holder_; - holder_ = NULL; - state_ = kNoObject; - ReadNextObject(); - if (state_ != kHaveObject) - return false; // eof or read error. - if (cur_key_.compare(last_key_) <= 0) { - KALDI_ERR << "You provided the \"s\" option " - << " (sorted order), but keys are out of order or duplicated: " - << last_key_ << " is followed by " << cur_key_ - << ": rspecifier is " << rspecifier_; - } - } - } - } - - /// Last string provided to HasKey() or Value(); - std::string last_requested_key_; - - -}; - -// RandomAccessTableReaderSortedArchiveImpl is for random-access reading of -// archives when the user specified the sorted (s) option but not the -// called-sorted (cs) options. -template<class Holder> class RandomAccessTableReaderSortedArchiveImpl: - public RandomAccessTableReaderArchiveImplBase<Holder> { - using RandomAccessTableReaderArchiveImplBase<Holder>::kUninitialized; - using RandomAccessTableReaderArchiveImplBase<Holder>::kHaveObject; - using RandomAccessTableReaderArchiveImplBase<Holder>::kNoObject; - using RandomAccessTableReaderArchiveImplBase<Holder>::kEof; - using RandomAccessTableReaderArchiveImplBase<Holder>::kError; - using RandomAccessTableReaderArchiveImplBase<Holder>::state_; - using RandomAccessTableReaderArchiveImplBase<Holder>::opts_; - using RandomAccessTableReaderArchiveImplBase<Holder>::cur_key_; - using RandomAccessTableReaderArchiveImplBase<Holder>::holder_; - using RandomAccessTableReaderArchiveImplBase<Holder>::rspecifier_; - using RandomAccessTableReaderArchiveImplBase<Holder>::archive_rxfilename_; - using RandomAccessTableReaderArchiveImplBase<Holder>::ReadNextObject; - - public: - typedef typename Holder::T T; - - RandomAccessTableReaderSortedArchiveImpl(): - last_found_index_(static_cast<size_t>(-1)), - pending_delete_(static_cast<size_t>(-1)) { } - - virtual bool Close() { - for (size_t i = 0; i < seen_pairs_.size(); i++) - if (seen_pairs_[i].second) - delete seen_pairs_[i].second; - seen_pairs_.clear(); - - pending_delete_ = static_cast<size_t>(-1); - last_found_index_ = static_cast<size_t>(-1); - - return this->CloseInternal(); - } - virtual bool HasKey(const std::string &key) { - HandlePendingDelete(); - size_t index; - bool ans = FindKeyInternal(key, &index); - if (ans && opts_.once && seen_pairs_[index].second == NULL) { - // Just do a check RE the once option. "&&opts_.once" is for - // efficiency since this can only happen in that case. - KALDI_ERR << "Error: HasKey called after Value() already called for " - << " that key, and once (o) option specified: rspecifier is " - << rspecifier_; - } - return ans; - } - virtual const T & Value(const std::string &key) { - HandlePendingDelete(); - size_t index; - if (FindKeyInternal(key, &index)) { - if (seen_pairs_[index].second == NULL) { // can happen if opts.once_ - KALDI_ERR << "Error: Value() called more than once for key " - << key << " and once (o) option specified: rspecifier is " - << rspecifier_; - } - if (opts_.once) - pending_delete_ = index; // mark this index to be deleted on next call. - return seen_pairs_[index].second->Value(); - } else { - KALDI_ERR << "Value() called but no such key " << key - << " in archive " << PrintableRxfilename(archive_rxfilename_); - return *(const T*)NULL; // keep compiler happy. - } - } - virtual ~RandomAccessTableReaderSortedArchiveImpl() { - if (this->IsOpen()) - if (!Close()) // more specific warning will already have been printed. - // we are in some kind of error state & user did not find out by - // calling Close(). - KALDI_ERR << "Error closing RandomAccessTableReader: rspecifier is " - << rspecifier_; - } - private: - void HandlePendingDelete() { - const size_t npos = static_cast<size_t>(-1); - if (pending_delete_ != npos) { - KALDI_ASSERT(pending_delete_ < seen_pairs_.size()); - KALDI_ASSERT(seen_pairs_[pending_delete_].second != NULL); - delete seen_pairs_[pending_delete_].second; - seen_pairs_[pending_delete_].second = NULL; - pending_delete_ = npos; - } - } - - // FindKeyInternal tries to find the key in the array "seen_pairs_". - // If it is not already there, it reads ahead as far as necessary - // to determine whether we have the key or not. On success it returns - // true and puts the index into the array seen_pairs_, into "index"; - // on failure it returns false. - // It will leave the state as either kNoObject, kEof or kError. - // FindKeyInternal does not do any checking about whether you are asking - // about a key that has been already given (with the "once" option). - // That is the user's responsibility. - - bool FindKeyInternal(const std::string &key, size_t *index) { - // First, an optimization in case the previous call was for the - // same key, and we found it. - if (last_found_index_ < seen_pairs_.size() - && seen_pairs_[last_found_index_].first == key) { - *index = last_found_index_; - return true; - } - - if (state_ == kUninitialized) - KALDI_ERR << "Trying to access a RandomAccessTableReader object that is not open."; - - // Step one is to see whether we have to read ahead for the object.. - // Note, the possible states right now are kNoObject, kEof or kError. - // We are never in the state kHaveObject except just after calling - // ReadNextObject(). - bool looped = false; - while (state_ == kNoObject && - (seen_pairs_.empty() || key.compare(seen_pairs_.back().first) > 0)) { - looped = true; - // Read this as: - // while ( the stream is potentially good for reading && - // ([got no keys] || key > most_recent_key) ) { ... - // Try to read a new object. - // Note that the keys in seen_pairs_ are ordered from least to greatest. - ReadNextObject(); - if (state_ == kHaveObject) { // Successfully read object. - if (!seen_pairs_.empty() && // This is just a check. - cur_key_.compare(seen_pairs_.back().first) <= 0) { - // read the expression above as: !( cur_key_ > previous_key). - // it means we are not in sorted order [the user specified that we - // are, or we would not be using this implementation]. - KALDI_ERR << "You provided the sorted (s) option but keys in archive " - << PrintableRxfilename(archive_rxfilename_) << " are not " - << "in sorted order: " << seen_pairs_.back().first - << " is followed by " << cur_key_; - } - KALDI_ASSERT(holder_ != NULL); - seen_pairs_.push_back(std::make_pair(cur_key_, holder_)); - holder_ = NULL; - state_ = kNoObject; - } - } - if (looped) { // We only need to check the last element of the seen_pairs_ array, - // since we would not have read more after getting "key". - if (!seen_pairs_.empty() && seen_pairs_.back().first == key) { - last_found_index_ = *index = seen_pairs_.size() - 1; - return true; - } else return false; - } - // Now we have do an actual binary search in the seen_pairs_ array. - std::pair<std::string, Holder*> pr(key, static_cast<Holder*>(NULL)); - typename std::vector<std::pair<std::string, Holder*> >::iterator - iter = std::lower_bound(seen_pairs_.begin(), seen_pairs_.end(), - pr, PairCompare()); - if (iter != seen_pairs_.end() && - key == iter->first) { - last_found_index_ = *index = (iter - seen_pairs_.begin()); - return true; - } else return false; - } - - // These are the pairs of (key, object) we have read. We keep all the keys we - // have read but the actual objects (if they are stored with pointers inside - // the Holder object) may be deallocated if once == true, and the Holder - // pointer set to NULL. - std::vector<std::pair<std::string, Holder*> > seen_pairs_; - size_t last_found_index_; // An optimization s.t. if FindKeyInternal called twice with - // same key (as it often will), it doesn't have to do the key search twice. - size_t pending_delete_; // If opts_.once == true, this is the index of - // element of seen_pairs_ that is pending deletion. - struct PairCompare { - // PairCompare is the Less-than operator for the pairs of(key, Holder). - // compares the keys. - inline bool operator() (const std::pair<std::string, Holder*> &pr1, - const std::pair<std::string, Holder*> &pr2) { - return (pr1.first.compare(pr2.first) < 0); - } - }; -}; - - - -// RandomAccessTableReaderUnsortedArchiveImpl is for random-access reading of -// archives when the user does not specify the sorted (s) option (in this case -// the called-sorted, or "cs" option, is ignored). This is the least efficient -// of the random access archive readers, in general, but it can be as efficient -// as the others, in speed, memory and latency, if the "once" option is specified -// and it happens that the keys of the archive are the same as the keys the code -// is called with (to HasKey() and Value()), and in the same order. However, if -// you ask it for a key that's not present it will have to read the archive till -// the end and store it all in memory. - -template<class Holder> class RandomAccessTableReaderUnsortedArchiveImpl: - public RandomAccessTableReaderArchiveImplBase<Holder> { - using RandomAccessTableReaderArchiveImplBase<Holder>::kUninitialized; - using RandomAccessTableReaderArchiveImplBase<Holder>::kHaveObject; - using RandomAccessTableReaderArchiveImplBase<Holder>::kNoObject; - using RandomAccessTableReaderArchiveImplBase<Holder>::kEof; - using RandomAccessTableReaderArchiveImplBase<Holder>::kError; - using RandomAccessTableReaderArchiveImplBase<Holder>::state_; - using RandomAccessTableReaderArchiveImplBase<Holder>::opts_; - using RandomAccessTableReaderArchiveImplBase<Holder>::cur_key_; - using RandomAccessTableReaderArchiveImplBase<Holder>::holder_; - using RandomAccessTableReaderArchiveImplBase<Holder>::rspecifier_; - using RandomAccessTableReaderArchiveImplBase<Holder>::archive_rxfilename_; - using RandomAccessTableReaderArchiveImplBase<Holder>::ReadNextObject; - - typedef typename Holder::T T; - - public: - RandomAccessTableReaderUnsortedArchiveImpl(): to_delete_iter_(map_.end()), - to_delete_iter_valid_(false) - { - map_.max_load_factor(0.5); // make it quite empty -> quite efficient. - // default seems to be 1. - } - - virtual bool Close() { - for (typename MapType::iterator iter = map_.begin(); - iter != map_.end(); - ++iter) { - if (iter->second) - delete iter->second; - } - map_.clear(); - first_deleted_string_ = ""; - to_delete_iter_valid_ = false; - return this->CloseInternal(); - } - - virtual bool HasKey(const std::string &key) { - HandlePendingDelete(); - return FindKeyInternal(key, NULL); - } - virtual const T & Value(const std::string &key) { - HandlePendingDelete(); - const T *ans_ptr = NULL; - if (FindKeyInternal(key, &ans_ptr)) - return *ans_ptr; - else - KALDI_ERR << "Value() called but no such key " << key - << " in archive " << PrintableRxfilename(archive_rxfilename_); - return *(const T*)NULL; // keep compiler happy. - } - virtual ~RandomAccessTableReaderUnsortedArchiveImpl() { - if (this->IsOpen()) - if (!Close()) // more specific warning will already have been printed. - // we are in some kind of error state & user did not find out by - // calling Close(). - KALDI_ERR << "Error closing RandomAccessTableReader: rspecifier is " - << rspecifier_; - } - private: - void HandlePendingDelete() { - if (to_delete_iter_valid_) { - to_delete_iter_valid_ = false; - delete to_delete_iter_->second; // Delete Holder object. - if (first_deleted_string_.length() == 0) - first_deleted_string_ = to_delete_iter_->first; - map_.erase(to_delete_iter_); // delete that element. - } - } - - // FindKeyInternal tries to find the key in the map "map_" - // If it is not already there, it reads ahead either until it finds the - // key, or until end of file. If called with value_ptr == NULL, - // it assumes it's called from HasKey() and just returns true or false - // and doesn't otherwise have side effects. If called with value_ptr != - // NULL, it assumes it's called from Value(). Thus, it will crash - // if it cannot find the key. If it can find it it puts its address in - // *value_ptr, and if opts_once == true it will mark that element of the - // map to be deleted. - - bool FindKeyInternal(const std::string &key, const T **value_ptr = NULL) { - typename MapType::iterator iter = map_.find(key); - if (iter != map_.end()) { // Found in the map... - if (value_ptr == NULL) { // called from HasKey - return true; // this is all we have to do. - } else { - *value_ptr = &(iter->second->Value()); - if (opts_.once) { // value won't be needed again, so mark - // for deletion. - to_delete_iter_ = iter; // pending delete. - KALDI_ASSERT(!to_delete_iter_valid_); - to_delete_iter_valid_ = true; - } - return true; - } - } - while (state_ == kNoObject) { - ReadNextObject(); - if (state_ == kHaveObject) { // Successfully read object. - state_ = kNoObject; // we are about to transfer ownership - // of the object in holder_ to map_. - // Insert it into map_. - std::pair<typename MapType::iterator, bool> pr = - map_.insert(typename MapType::value_type(cur_key_, holder_)); - - if (!pr.second) { // Was not inserted-- previous element w/ same key - delete holder_; // map was not changed, no ownership transferred. - holder_ = NULL; - KALDI_ERR << "Error in RandomAccessTableReader: duplicate key " - << cur_key_ << " in archive " << archive_rxfilename_; - } - holder_ = NULL; // ownership transferred to map_. - if (cur_key_ == key) { // the one we wanted.. - if (value_ptr == NULL) { // called from HasKey - return true; - } else { // called from Value() - *value_ptr = &(pr.first->second->Value()); // this gives us the - // Value() from the Holder in the map. - if (opts_.once) { // mark for deletion, as won't be needed again. - to_delete_iter_ = pr.first; - KALDI_ASSERT(!to_delete_iter_valid_); - to_delete_iter_valid_ = true; - } - return true; - } - } - } - } - if (opts_.once && key == first_deleted_string_) { - KALDI_ERR << "You specified the once (o) option but " - << "you are calling using key " << key - << " more than once: rspecifier is " << rspecifier_; - } - return false; // We read the entire archive (or got to error state) and didn't - // find it. - } - - typedef unordered_map<std::string, Holder*, StringHasher> MapType; - MapType map_; - - typename MapType::iterator to_delete_iter_; - bool to_delete_iter_valid_; - - std::string first_deleted_string_; // keep the first string we deleted - // from map_ (if opts_.once == true). It's for an inexact spot-check that the - // "once" option isn't being used incorrectly. - -}; - - - - - -template<class Holder> -RandomAccessTableReader<Holder>::RandomAccessTableReader(const std::string &rspecifier): - impl_(NULL) { - if (rspecifier != "" && !Open(rspecifier)) - KALDI_ERR << "Error opening RandomAccessTableReader object " - " (rspecifier is: " << rspecifier << ")"; -} - -template<class Holder> -bool RandomAccessTableReader<Holder>::Open(const std::string &rspecifier) { - if (IsOpen()) - KALDI_ERR << "Already open."; - RspecifierOptions opts; - RspecifierType rs = ClassifyRspecifier(rspecifier, NULL, &opts); - switch (rs) { - case kScriptRspecifier: - impl_ = new RandomAccessTableReaderScriptImpl<Holder>(); - break; - case kArchiveRspecifier: - if (opts.sorted) { - if (opts.called_sorted) // "doubly" sorted case. - impl_ = new RandomAccessTableReaderDSortedArchiveImpl<Holder>(); - else - impl_ = new RandomAccessTableReaderSortedArchiveImpl<Holder>(); - } else impl_ = new RandomAccessTableReaderUnsortedArchiveImpl<Holder>(); - break; - case kNoRspecifier: default: - KALDI_WARN << "Invalid rspecifier: " - << rspecifier; - return false; - } - if (impl_->Open(rspecifier)) - return true; - else { - // Warning will already have been printed. - delete impl_; - impl_ = NULL; - return false; - } -} - -template<class Holder> -bool RandomAccessTableReader<Holder>::HasKey(const std::string &key) { - CheckImpl(); - if (!IsToken(key)) - KALDI_ERR << "Invalid key \"" << key << '"'; - return impl_->HasKey(key); -} - - -template<class Holder> -const typename RandomAccessTableReader<Holder>::T& -RandomAccessTableReader<Holder>::Value(const std::string &key) { - CheckImpl(); - return impl_->Value(key); -} - -template<class Holder> -bool RandomAccessTableReader<Holder>::Close() { - CheckImpl(); - bool ans =impl_->Close(); - delete impl_; - impl_ = NULL; - return ans; -} - -template<class Holder> -RandomAccessTableReader<Holder>::~RandomAccessTableReader() { - if (IsOpen() && !Close()) // call Close() yourself to stop this being thrown. - KALDI_ERR << "failure detected in destructor."; -} - -template<class Holder> -void SequentialTableReader<Holder>::CheckImpl() const { - if (!impl_) { - KALDI_ERR << "Trying to use empty SequentialTableReader (perhaps you " - << "passed the empty string as an argument to a program?)"; - } -} - -template<class Holder> -void RandomAccessTableReader<Holder>::CheckImpl() const { - if (!impl_) { - KALDI_ERR << "Trying to use empty RandomAccessTableReader (perhaps you " - << "passed the empty string as an argument to a program?)"; - } -} - -template<class Holder> -void TableWriter<Holder>::CheckImpl() const { - if (!impl_) { - KALDI_ERR << "Trying to use empty TableWriter (perhaps you " - << "passed the empty string as an argument to a program?)"; - } -} - -template<class Holder> -RandomAccessTableReaderMapped<Holder>::RandomAccessTableReaderMapped( - const std::string &table_rxfilename, - const std::string &utt2spk_rxfilename): - reader_(table_rxfilename), token_reader_(table_rxfilename.empty() ? "" : - utt2spk_rxfilename), - utt2spk_rxfilename_(utt2spk_rxfilename) { } - -template<class Holder> -bool RandomAccessTableReaderMapped<Holder>::Open( - const std::string &table_rxfilename, - const std::string &utt2spk_rxfilename) { - if (reader_.IsOpen()) reader_.Close(); - if (token_reader_.IsOpen()) token_reader_.Close(); - KALDI_ASSERT(!table_rxfilename.empty()); - if (!reader_.Open(table_rxfilename)) return false; // will have printed - // warning internally, probably. - if (!utt2spk_rxfilename.empty()) { - if (!token_reader_.Open(utt2spk_rxfilename)) { - reader_.Close(); - return false; - } - } - return true; -} - - -template<class Holder> -bool RandomAccessTableReaderMapped<Holder>::HasKey(const std::string &utt) { - // We don't check IsOpen, we let the call go through to the member variable - // (reader_), which will crash with a more informative error message than - // we can give here, as we don't any longer know the rxfilename. - if (token_reader_.IsOpen()) { // We need to map the key from utt to spk. - if (!token_reader_.HasKey(utt)) - KALDI_ERR << "Attempting to read key " << utt << ", which is not present " - << "in utt2spk map or similar map being read from " - << PrintableRxfilename(utt2spk_rxfilename_); - const std::string &spk = token_reader_.Value(utt); - return reader_.HasKey(spk); - } else { - return reader_.HasKey(utt); - } -} - -template<class Holder> -const typename Holder::T& RandomAccessTableReaderMapped<Holder>::Value( - const std::string &utt) { - if (token_reader_.IsOpen()) { // We need to map the key from utt to spk. - if (!token_reader_.HasKey(utt)) - KALDI_ERR << "Attempting to read key " << utt << ", which is not present " - << "in utt2spk map or similar map being read from " - << PrintableRxfilename(utt2spk_rxfilename_); - const std::string &spk = token_reader_.Value(utt); - return reader_.Value(spk); - } else { - return reader_.Value(utt); - } -} - - - -/// @} - -} // end namespace kaldi - - - -#endif |