summaryrefslogtreecommitdiff
path: root/kaldi_decode
diff options
context:
space:
mode:
authorDeterminant <[email protected]>2016-02-29 14:38:33 +0800
committerDeterminant <[email protected]>2016-02-29 14:38:33 +0800
commitfda1c8cf07c5130aff53775454a5f2cfc8f5d2e0 (patch)
tree2ec85df4011268c78fa5eee9c9f09f9b05eafafc /kaldi_decode
parent239d7d8609ce4d1b5cbe7468506758d045a0995f (diff)
save workspace
Diffstat (limited to 'kaldi_decode')
-rwxr-xr-xkaldi_decode/cmd.sh3
-rwxr-xr-xkaldi_decode/decode.sh124
-rwxr-xr-xkaldi_decode/path.sh12
-rw-r--r--kaldi_decode/src/Makefile6
-rw-r--r--kaldi_decode/src/nerv4decode.lua27
5 files changed, 31 insertions, 141 deletions
diff --git a/kaldi_decode/cmd.sh b/kaldi_decode/cmd.sh
index e2e54e8..be10905 100755
--- a/kaldi_decode/cmd.sh
+++ b/kaldi_decode/cmd.sh
@@ -25,7 +25,8 @@
#c) run it locally...
export train_cmd=run.pl
-export decode_cmd=run.pl
+#export decode_cmd=run.pl
+export decode_cmd='queue.pl -l hostname="markov"'
export cuda_cmd=run.pl
export mkgraph_cmd=run.pl
diff --git a/kaldi_decode/decode.sh b/kaldi_decode/decode.sh
deleted file mode 100755
index aa7e089..0000000
--- a/kaldi_decode/decode.sh
+++ /dev/null
@@ -1,124 +0,0 @@
-#!/bin/bash
-
-# Copyright 2012-2013 Karel Vesely, Daniel Povey
-# Apache 2.0
-
-# Begin configuration section.
-nnet= # non-default location of DNN (optional)
-feature_transform= # non-default location of feature_transform (optional)
-model= # non-default location of transition model (optional)
-class_frame_counts= # non-default location of PDF counts (optional)
-srcdir= # non-default location of DNN-dir (decouples model dir from decode dir)
-
-stage=0 # stage=1 skips lattice generation
-nj=4
-cmd=run.pl
-
-acwt=0.10 # note: only really affects pruning (scoring is on lattices).
-beam=13.0
-lattice_beam=8.0
-min_active=200
-max_active=7000 # limit of active tokens
-max_mem=50000000 # approx. limit to memory consumption during minimization in bytes
-nnet_forward_opts="--prior-scale=1.0"
-
-skip_scoring=false
-scoring_opts="--min-lmwt 4 --max-lmwt 15"
-
-num_threads=1 # if >1, will use latgen-faster-parallel
-parallel_opts= # Ignored now.
-use_gpu="no" # yes|no|optionaly
-# End configuration section.
-
-echo "$0 $@" # Print the command line for logging
-
-[ -f ./path.sh ] && . ./path.sh; # source the path.
-. parse_options.sh || exit 1;
-
-if [ $# != 4 ]; then
- echo "Usage: $0 [options] <graph-dir> <data-dir> <nerv-config> <decode-dir>"
- echo "... where <decode-dir> is assumed to be a sub-directory of the directory"
- echo " where the DNN and transition model is."
- echo "e.g.: $0 exp/dnn1/graph_tgpr data/test config.lua exp/dnn1/decode_tgpr"
- echo ""
- echo "This script works on plain or modified features (CMN,delta+delta-delta),"
- echo "which are then sent through feature-transform. It works out what type"
- echo "of features you used from content of srcdir."
- echo ""
- echo "main options (for others, see top of script file)"
- echo " --config <config-file> # config containing options"
- echo " --nj <nj> # number of parallel jobs"
- echo " --cmd (utils/run.pl|utils/queue.pl <queue opts>) # how to run jobs."
- echo ""
- echo " --srcdir <dir> # non-default dir with DNN/models, can be different"
- echo " # from parent dir of <decode-dir>' (opt.)"
- echo ""
- echo " --acwt <float> # select acoustic scale for decoding"
- echo " --scoring-opts <opts> # options forwarded to local/score.sh"
- echo " --num-threads <N> # N>1: run multi-threaded decoder"
- exit 1;
-fi
-
-
-graphdir=$1
-data=$2
-config=$3
-dir=$4
-[ -z $srcdir ] && srcdir=`dirname $dir`; # Default model directory one level up from decoding directory.
-sdata=$data/split$nj;
-
-mkdir -p $dir/log
-
-[[ -d $sdata && $data/feats.scp -ot $sdata ]] || split_data.sh $data $nj || exit 1;
-echo $nj > $dir/num_jobs
-
-# Select default locations to model files (if not already set externally)
-[ -z "$model" ] && model=$srcdir/final.mdl
-#
-[ -z "$class_frame_counts" -a -f $srcdir/prior_counts ] && class_frame_counts=$srcdir/prior_counts # priority,
-[ -z "$class_frame_counts" ] && class_frame_counts=$srcdir/ali_train_pdf.counts
-
-# Check that files exist
-for f in $sdata/1/feats.scp $model $class_frame_counts $graphdir/HCLG.fst; do
- [ ! -f $f ] && echo "$0: missing file $f" && exit 1;
-done
-
-# Possibly use multi-threaded decoder
-thread_string=
-[ $num_threads -gt 1 ] && thread_string="-parallel --num-threads=$num_threads"
-
-
-# PREPARE FEATURE EXTRACTION PIPELINE
-# import config,
-cmvn_opts=
-delta_opts=
-D=$srcdir
-[ -e $D/norm_vars ] && cmvn_opts="--norm-means=true --norm-vars=$(cat $D/norm_vars)" # Bwd-compatibility,
-[ -e $D/cmvn_opts ] && cmvn_opts=$(cat $D/cmvn_opts)
-[ -e $D/delta_order ] && delta_opts="--delta-order=$(cat $D/delta_order)" # Bwd-compatibility,
-[ -e $D/delta_opts ] && delta_opts=$(cat $D/delta_opts)
-#
-# Create the feature stream,
-feats="ark,s,cs:copy-feats scp:$sdata/JOB/feats.scp ark:- |"
-# apply-cmvn (optional),
-[ ! -z "$cmvn_opts" -a ! -f $sdata/1/cmvn.scp ] && echo "$0: Missing $sdata/1/cmvn.scp" && exit 1
-[ ! -z "$cmvn_opts" ] && feats="$feats apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp ark:- ark:- |"
-# add-deltas (optional),
-[ ! -z "$delta_opts" ] && feats="$feats add-deltas $delta_opts ark:- ark:- |"
-#
-# Run the decoding in the queue,
-if [ $stage -le 0 ]; then
- $cmd --num-threads $((num_threads+1)) JOB=1:$nj $dir/log/decode.JOB.log \
- ./src/nnet-forward $nnet_forward_opts --class-frame-counts=$class_frame_counts --use-gpu=$use_gpu $config "$feats" ark:- \| \
- latgen-faster-mapped$thread_string --min-active=$min_active --max-active=$max_active --max-mem=$max_mem --beam=$beam \
- --lattice-beam=$lattice_beam --acoustic-scale=$acwt --allow-partial=true --word-symbol-table=$graphdir/words.txt \
- $model $graphdir/HCLG.fst ark:- "ark:|gzip -c > $dir/lat.JOB.gz" || exit 1;
-fi
-# Run the scoring
-if ! $skip_scoring ; then
- [ ! -x local/score.sh ] && \
- echo "Not scoring because local/score.sh does not exist or not executable." && exit 1;
- local/score.sh $scoring_opts --cmd "$cmd" $data $graphdir $dir || exit 1;
-fi
-
-exit 0;
diff --git a/kaldi_decode/path.sh b/kaldi_decode/path.sh
index 5aebc72..5e9bd2a 100755
--- a/kaldi_decode/path.sh
+++ b/kaldi_decode/path.sh
@@ -1,5 +1,11 @@
-export KALDI_ROOT=/slfs6/users/ymz09/kaldi/
-export PATH=$PWD/utils/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/src/fstbin/:$KALDI_ROOT/src/gmmbin/:$KALDI_ROOT/src/featbin/:$KALDI_ROOT/src/lm/:$KALDI_ROOT/src/sgmmbin/:$KALDI_ROOT/src/sgmm2bin/:$KALDI_ROOT/src/fgmmbin/:$KALDI_ROOT/src/latbin/:$KALDI_ROOT/src/nnetbin:$KALDI_ROOT/src/nnet2bin/:$KALDI_ROOT/src/kwsbin:$PWD:$PATH
+### change this line to your kaldi repo
+export KALDI_ROOT=/speechlab/tools/KALDI/kaldi-master/
+### the following lines should not be changed in most cases
+
+# setup kaldi path
+[ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh
+export PATH=$PWD/utils/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/tools/irstlm/bin/:$KALDI_ROOT/src/fstbin/:$KALDI_ROOT/src/gmmbin/:$KALDI_ROOT/src/featbin/:$KALDI_ROOT/src/lm/:$KALDI_ROOT/src/sgmmbin/:$KALDI_ROOT/src/sgmm2bin/:$KALDI_ROOT/src/fgmmbin/:$KALDI_ROOT/src/latbin/:$KALDI_ROOT/src/nnetbin:$KALDI_ROOT/src/nnet2bin/:$KALDI_ROOT/src/kwsbin:$PWD:$PATH
export LC_ALL=C
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/slhome/ymz09/mylibs/:.
+# setup luarocks path and cpath for NERV (important)
+source <(../../install/bin/luarocks path)
diff --git a/kaldi_decode/src/Makefile b/kaldi_decode/src/Makefile
index 118420c..0897798 100644
--- a/kaldi_decode/src/Makefile
+++ b/kaldi_decode/src/Makefile
@@ -1,11 +1,11 @@
# Change KDIR to `kaldi-trunk' path (Kaldi must be compiled with --share)
-KDIR := /slfs6/users/ymz09/kaldi/
-NERVDIR := /slfs6/users/ymz09/nerv-project/nerv/
+KDIR := /speechlab/tools/KALDI/kaldi-master/
+NERVDIR := /speechlab/users/mfy43/nerv/
CUDADIR := /usr/local/cuda/
nnet-forward:
g++ -msse -msse2 -Wall -I $(KDIR)/src/ -pthread -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -Wno-sign-compare -Wno-unused-local-typedefs -Winit-self -DHAVE_EXECINFO_H=1 -rdynamic -DHAVE_CXXABI_H -DHAVE_ATLAS -I $(KDIR)/tools/ATLAS/include -I $(KDIR)/tools/openfst/include -Wno-sign-compare -g -fPIC -DHAVE_CUDA -I $(CUDADIR)/include -DKALDI_NO_EXPF -I $(NERVDIR)/install//include/luajit-2.0/ -I $(NERVDIR)/install/include/ -DLUA_USE_APICHECK -c -o nnet-forward.o nnet-forward.cc
- g++ -rdynamic -Wl,-rpath=$(KDIR)/tools/openfst/lib -L$(CUDADIR)/lib64 -Wl,-rpath,$(CUDADIR)/lib64 -Wl,-rpath=$(KDIR)/src/lib -L. -L$(KDIR)/src/nnet/ -L$(KDIR)/src/cudamatrix/ -L$(KDIR)/src/lat/ -L$(KDIR)/src/hmm/ -L$(KDIR)/src/tree/ -L$(KDIR)/src/matrix/ -L$(KDIR)/src/util/ -L$(KDIR)/src/base/ nnet-forward.o $(KDIR)/src/nnet//libkaldi-nnet.so $(KDIR)/src/cudamatrix//libkaldi-cudamatrix.so $(KDIR)/src/lat//libkaldi-lat.so $(KDIR)/src/hmm//libkaldi-hmm.so $(KDIR)/src/tree//libkaldi-tree.so $(KDIR)/src/matrix//libkaldi-matrix.so $(KDIR)/src/util//libkaldi-util.so $(KDIR)/src/base//libkaldi-base.so -L$(KDIR)/tools/openfst/lib -lfst /usr/lib/liblapack.so /usr/lib/libcblas.so /usr/lib/libatlas.so /usr/lib/libf77blas.so -lm -lpthread -ldl -lcublas -lcudart -lkaldi-nnet -lkaldi-cudamatrix -lkaldi-lat -lkaldi-hmm -lkaldi-tree -lkaldi-matrix -lkaldi-util -lkaldi-base -lstdc++ -L$(NERVDIR)/install/lib -lnervcore -lluaT -rdynamic -Wl,-rpath=$(KDIR)//tools/openfst/lib -L$(DUDADIR)/lib64 -Wl,-rpath,$(CUDADIR)/lib64 -Wl,-rpath=$(KDIR)//src/lib -lfst -lm -lpthread -ldl -lcublas -lcudart -L $(NERVDIR)/luajit-2.0/src/ -lluajit -o nnet-forward
+ g++ -rdynamic -Wl,-rpath=$(KDIR)/tools/openfst/lib -L$(CUDADIR)/lib64 -Wl,-rpath,$(CUDADIR)/lib64 -Wl,-rpath=$(KDIR)/src/lib -L. -L$(KDIR)/src/nnet/ -L$(KDIR)/src/cudamatrix/ -L$(KDIR)/src/lat/ -L$(KDIR)/src/hmm/ -L$(KDIR)/src/tree/ -L$(KDIR)/src/matrix/ -L$(KDIR)/src/util/ -L$(KDIR)/src/base/ nnet-forward.o $(KDIR)/src/nnet//libkaldi-nnet.so $(KDIR)/src/cudamatrix//libkaldi-cudamatrix.so $(KDIR)/src/lat//libkaldi-lat.so $(KDIR)/src/hmm//libkaldi-hmm.so $(KDIR)/src/tree//libkaldi-tree.so $(KDIR)/src/matrix//libkaldi-matrix.so $(KDIR)/src/util//libkaldi-util.so $(KDIR)/src/base//libkaldi-base.so -L$(KDIR)/tools/openfst/lib -lfst -lm -lpthread -ldl -lkaldi-nnet -lkaldi-cudamatrix -lkaldi-lat -lkaldi-hmm -lkaldi-tree -lkaldi-matrix -lkaldi-util -lkaldi-base -lstdc++ -L$(NERVDIR)/install/lib -Wl,-rpath=$(NERVDIR)/install/lib -lnervcore -lluaT -rdynamic -Wl,-rpath=$(KDIR)//tools/openfst/lib -L$(DUDADIR)/lib64 -Wl,-rpath,$(CUDADIR)/lib64 -Wl,-rpath=$(KDIR)//src/lib -lfst -lm -lpthread -ldl -L $(NERVDIR)/luajit-2.0/src/ -lluajit -o nnet-forward -L/home/intel/mkl/lib/intel64/ -Wl,-rpath=/home/intel/mkl/lib/intel64/ -lmkl_rt
clean:
-rm nnet-forward.o nnet-forward
diff --git a/kaldi_decode/src/nerv4decode.lua b/kaldi_decode/src/nerv4decode.lua
index b2ff344..898b5a8 100644
--- a/kaldi_decode/src/nerv4decode.lua
+++ b/kaldi_decode/src/nerv4decode.lua
@@ -1,7 +1,19 @@
-package.path="/home/slhome/ymz09/.luarocks/share/lua/5.1/?.lua;/home/slhome/ymz09/.luarocks/share/lua/5.1/?/init.lua;/slfs6/users/ymz09/nerv-project/nerv/install/share/lua/5.1/?.lua;/slfs6/users/ymz09/nerv-project/nerv/install/share/lua/5.1/?/init.lua;"..package.path;
-package.cpath="/home/slhome/ymz09/.luarocks/lib/lua/5.1/?.so;/slfs6/users/ymz09/nerv-project/nerv/install/lib/lua/5.1/?.so;"..package.cpath;
+print = function(...) io.write(table.concat({...}, "\t")) end
+io.output('/dev/null')
+-- path and cpath are correctly set by `path.sh`
local k,l,_=pcall(require,"luarocks.loader") _=k and l.add_context("nerv","scm-1")
require 'nerv'
+nerv.printf("*** NERV: A Lua-based toolkit for high-performance deep learning (alpha) ***\n")
+nerv.info("automatically initialize a default MContext...")
+nerv.MMatrix._default_context = nerv.MContext()
+nerv.info("the default MContext is ok")
+-- only for backward compatibilty, will be removed in the future
+local function _add_profile_method(cls)
+ local c = cls._default_context
+ cls.print_profile = function () c:print_profile() end
+ cls.clear_profile = function () c:clear_profile() end
+end
+_add_profile_method(nerv.MMatrix)
function build_trainer(ifname, feature)
local param_repo = nerv.ParamRepo()
@@ -34,9 +46,7 @@ function build_trainer(ifname, feature)
end
local transformed
if e.global_transf then
- local batch = gconf.cumat_type(data[id]:nrow(), data[id]:ncol())
- batch:copy_fromh(data[id])
- transformed = nerv.speech_utils.global_transf(batch,
+ transformed = nerv.speech_utils.global_transf(data[id],
global_transf,
gconf.frm_ext or 0, 0,
gconf)
@@ -45,7 +55,7 @@ function build_trainer(ifname, feature)
end
table.insert(input, transformed)
end
- local output = {nerv.CuMatrixFloat(input[1]:nrow(), network.dim_out[1])}
+ local output = {nerv.MMatrixFloat(input[1]:nrow(), network.dim_out[1])}
network:batch_resize(input[1]:nrow())
network:propagate(input, output)
@@ -65,12 +75,9 @@ function build_trainer(ifname, feature)
end
function init(config, feature)
- local tmp = io.write
- io.write = function(...)
- end
dofile(config)
+ gconf.use_cpu = true -- use CPU to decode
trainer = build_trainer(gconf.decode_param, feature)
- io.write = tmp
end
function feed()