From 1e0ac0fb5c9f517e7325deb16004de1054454da7 Mon Sep 17 00:00:00 2001 From: Determinant Date: Mon, 29 Feb 2016 20:03:52 +0800 Subject: refactor kaldi_decode --- htk_io/Makefile | 4 + kaldi_decode/Makefile | 43 +++++++++ kaldi_decode/README | 13 --- kaldi_decode/README.timit | 15 +++ kaldi_decode/cmd.sh | 37 ------- kaldi_decode/conf/decode_dnn.config | 2 - kaldi_decode/decode_with_nerv.sh | 134 ++++++++++++++++++++++++++ kaldi_decode/kaldi_decode-scm-1.rockspec | 36 +++++++ kaldi_decode/local/score.sh | 67 ------------- kaldi_decode/nnet-forward-with-nerv.sh | 2 + kaldi_decode/path.sh | 11 --- kaldi_decode/src/Makefile | 12 --- kaldi_decode/src/asr_propagator.lua | 84 ++++++++++++++++ kaldi_decode/src/nerv4decode.lua | 86 ----------------- kaldi_decode/src/nnet-forward.cc | 12 +-- kaldi_io/Makefile | 11 ++- kaldi_io/kaldi_io-scm-1.rockspec | 2 +- kaldi_io/tools/convert_from_kaldi_pretrain.sh | 66 +++++++++++++ kaldi_io/tools/kaldi_to_nerv | Bin 0 -> 18442 bytes kaldi_seq/Makefile | 10 +- tutorial/howto_pretrain_from_kaldi.rst | 60 ++++++++++++ 21 files changed, 467 insertions(+), 240 deletions(-) create mode 100644 kaldi_decode/Makefile delete mode 100755 kaldi_decode/README create mode 100755 kaldi_decode/README.timit delete mode 100755 kaldi_decode/cmd.sh delete mode 100644 kaldi_decode/conf/decode_dnn.config create mode 100755 kaldi_decode/decode_with_nerv.sh create mode 100644 kaldi_decode/kaldi_decode-scm-1.rockspec delete mode 100755 kaldi_decode/local/score.sh create mode 100644 kaldi_decode/nnet-forward-with-nerv.sh delete mode 100755 kaldi_decode/path.sh delete mode 100644 kaldi_decode/src/Makefile create mode 100644 kaldi_decode/src/asr_propagator.lua delete mode 100644 kaldi_decode/src/nerv4decode.lua create mode 100755 kaldi_io/tools/convert_from_kaldi_pretrain.sh create mode 100755 kaldi_io/tools/kaldi_to_nerv create mode 100644 tutorial/howto_pretrain_from_kaldi.rst diff --git a/htk_io/Makefile b/htk_io/Makefile index 45ed5a9..6a5f529 100644 --- a/htk_io/Makefile +++ b/htk_io/Makefile @@ -1,3 +1,7 @@ +ifndef LUA_BINDIR +$(error Please build the package via luarocks: `luarocks make`) +endif + .PHONY: tnet SHELL := /bin/bash BUILD_DIR := $(CURDIR)/build diff --git a/kaldi_decode/Makefile b/kaldi_decode/Makefile new file mode 100644 index 0000000..e3a7c2d --- /dev/null +++ b/kaldi_decode/Makefile @@ -0,0 +1,43 @@ +ifndef LUA_BINDIR +$(error Please build the package via luarocks: `luarocks make`) +endif + +ifndef KALDI_BASE +$(error KALDI_BASE is not set) +endif + +ifndef CUDA_BASE +$(error CUDA_BASE is not set) +endif + +KDIR := $(KALDI_BASE) +BUILD_DIR := $(CURDIR)/build +INC_PATH := $(LUA_BINDIR)/../include/ +OBJS := src/nnet-forward.o nnet-forward + +SUBDIR := src +OBJ_DIR := $(BUILD_DIR)/objs +LUA_DIR = $(INST_LUADIR)/kaldi_decode +KALDIINCLUDE := -I $(KDIR)/tools/ATLAS/include/ -I $(KDIR)/tools/openfst/include/ -I $(KDIR)/src/ + +OBJS := $(addprefix $(OBJ_DIR)/,$(OBJS)) +OBJ_SUBDIR := $(addprefix $(OBJ_DIR)/,$(SUBDIR)) + +KL := $(KDIR)/src/feat/kaldi-feat.a $(KDIR)/src/cudamatrix/kaldi-cudamatrix.a $(KDIR)/src/matrix/kaldi-matrix.a $(KDIR)/src/base/kaldi-base.a $(KDIR)/src/util/kaldi-util.a $(KDIR)/src/hmm/kaldi-hmm.a $(KDIR)/src/tree/kaldi-tree.a $(KDIR)/src/nnet/kaldi-nnet.a $(BLAS_LDFLAGS) + +build: $(OBJ_DIR) $(LUA_DIR) $(OBJ_SUBDIR) $(OBJS) +$(OBJ_DIR)/%.o: %.cc + g++ -c -o $@ $< -Wall $(KALDIINCLUDE) -DHAVE_ATLAS -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DLUA_USE_APICHECK -I $(LUA_INCDIR) -I $(INC_PATH) $(CFLAGS) +$(OBJ_DIR)/nnet-forward: $(OBJ_DIR)/src/nnet-forward.o + g++ -o $@ $< $(KL) -L$(LUA_LIBDIR) -Wl,-rpath=$(LUA_LIBDIR) -lluajit-5.1 -L$(CUDA_BASE)/lib64/ -Wl,-rpath=$(CUDA_BASE)/lib64/ -lcudart -lcublas -ldl +$(OBJ_DIR) $(LUA_DIR) $(OBJ_SUBDIR): + -mkdir -p $@ +install: $(LUA_DIR) + cp $(OBJ_DIR)/nnet-forward $(LUA_BINDIR)/nnet-forward-with-nerv + cp src/asr_propagator.lua $(LUA_DIR)/ + sed 's*nnet_forward=*nnet_forward=$(LUA_BINDIR)/nnet-forward-with-nerv.sh*g;s*asr_propagator=*asr_propagator=$(LUA_BINDIR)/../share/lua/5.1/kaldi_decode/asr_propagator.lua*g' decode_with_nerv.sh > $(LUA_BINDIR)/decode_with_nerv.sh + echo '$(LUA_BINDIR)/nnet-forward-with-nerv "$$@"' | cat nnet-forward-with-nerv.sh - | sed 's*\.\./\.\./install/bin/luarocks*$(LUA_BINDIR)/luarocks*g' > $(LUA_BINDIR)/nnet-forward-with-nerv.sh + chmod +x $(LUA_BINDIR)/nnet-forward-with-nerv.sh + chmod +x $(LUA_BINDIR)/decode_with_nerv.sh +clean: + -rm -r $(OBJ_DIR) diff --git a/kaldi_decode/README b/kaldi_decode/README deleted file mode 100755 index 8d0a95b..0000000 --- a/kaldi_decode/README +++ /dev/null @@ -1,13 +0,0 @@ -source path.sh -source cmd.sh - -acwt=0.1 -dir=/slfs5/users/ymz09/chime/baseline/ASR/exp/nerv_seq/ -graph=/slfs5/users/ymz09/chime/baseline/ASR/exp/tri4a_dnn_tr05_multi_enhanced/graph_tgpr_5k -data=/slfs5/users/ymz09/chime/baseline/ASR/data-fbank/et05_real_enhanced -config=/slfs6/users/ymz09/nerv-project/nerv/nerv/examples/mpe_chime3.lua - -decode.sh --nj 4 --cmd "$decode_cmd" --config conf/decode_dnn.config --acwt $acwt \ - $graph $data $config \ - $dir/decode_tgpr_5k_et05_real_enhanced_nerv - diff --git a/kaldi_decode/README.timit b/kaldi_decode/README.timit new file mode 100755 index 0000000..7fac918 --- /dev/null +++ b/kaldi_decode/README.timit @@ -0,0 +1,15 @@ +#!/bin/bash +source path.sh +source cmd.sh + +gmmdir=/speechlab/users/mfy43/timit/s5/exp/tri3/ +data_fmllr=/speechlab/users/mfy43/timit/s5/data-fmllr-tri3/ +dir=/speechlab/users/mfy43/timit/s5/exp/dnn4_nerv_prepare/ +nerv_config=/speechlab/users/mfy43/nerv/nerv/examples/timit_baseline2.lua +decode=/speechlab/users/mfy43/nerv/install/bin/decode_with_nerv.sh + +# Decode (reuse HCLG graph) +$decode --nj 20 --cmd "$decode_cmd" --acwt 0.2 \ + $gmmdir/graph $data_fmllr/test $nerv_config $dir/decode_test || exit 1; +$decode --nj 20 --cmd "$decode_cmd" --acwt 0.2 \ + $gmmdir/graph $data_fmllr/dev $nerv_config $dir/decode_dev || exit 1; diff --git a/kaldi_decode/cmd.sh b/kaldi_decode/cmd.sh deleted file mode 100755 index be10905..0000000 --- a/kaldi_decode/cmd.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# "queue.pl" uses qsub. The options to it are -# options to qsub. If you have GridEngine installed, -# change this to a queue you have access to. -# Otherwise, use "run.pl", which will run jobs locally -# (make sure your --num-jobs options are no more than -# the number of cpus on your machine. - -#a) JHU cluster options -#export train_cmd="queue.pl -l arch=*64" -#export decode_cmd="queue.pl -l arch=*64,mem_free=2G,ram_free=2G" -#export mkgraph_cmd="queue.pl -l arch=*64,ram_free=4G,mem_free=4G" - -#export cuda_cmd="..." - - -#b) BUT cluster options -#export train_cmd="queue.pl -q all.q@@blade -l ram_free=1200M,mem_free=1200M" -#export decode_cmd="queue.pl -q all.q@@blade -l ram_free=1700M,mem_free=1700M" -#export decodebig_cmd="queue.pl -q all.q@@blade -l ram_free=4G,mem_free=4G" - -#export cuda_cmd="queue.pl -q long.q@@pco203 -l gpu=1" -#export cuda_cmd="queue.pl -q long.q@pcspeech-gpu" -#export mkgraph_cmd="queue.pl -q all.q@@servers -l ram_free=4G,mem_free=4G" - -#c) run it locally... -export train_cmd=run.pl -#export decode_cmd=run.pl -export decode_cmd='queue.pl -l hostname="markov"' -export cuda_cmd=run.pl -export mkgraph_cmd=run.pl - -#export train_cmd='queue.pl' -#export decode_cmd='queue.pl' -#export cuda_cmd='queue.pl -l gpu=1 -l hostname="markov|date|hamming"' -#export mkgraph_cmd='queue.pl"' - diff --git a/kaldi_decode/conf/decode_dnn.config b/kaldi_decode/conf/decode_dnn.config deleted file mode 100644 index 89dd992..0000000 --- a/kaldi_decode/conf/decode_dnn.config +++ /dev/null @@ -1,2 +0,0 @@ -beam=18.0 # beam for decoding. Was 13.0 in the scripts. -lattice_beam=10.0 # this has most effect on size of the lattices. diff --git a/kaldi_decode/decode_with_nerv.sh b/kaldi_decode/decode_with_nerv.sh new file mode 100755 index 0000000..5554b2e --- /dev/null +++ b/kaldi_decode/decode_with_nerv.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# Copyright 2012-2013 Karel Vesely, Daniel Povey +# Apache 2.0 + +# Begin configuration section. +nnet= # non-default location of DNN (optional) +feature_transform= # non-default location of feature_transform (optional) +model= # non-default location of transition model (optional) +class_frame_counts= # non-default location of PDF counts (optional) +srcdir= # non-default location of DNN-dir (decouples model dir from decode dir) + +stage=0 # stage=1 skips lattice generation +nj=4 +cmd=run.pl + +acwt=0.10 # note: only really affects pruning (scoring is on lattices). +beam=13.0 +lattice_beam=8.0 +min_active=200 +max_active=7000 # limit of active tokens +max_mem=50000000 # approx. limit to memory consumption during minimization in bytes +nnet_forward_opts="--apply-log=true" # IMPORTANT, to apply log before to substract log-prior, and to know the modified 'nnet-forward' removed '--no-softmax' option + +skip_scoring=false +scoring_opts="--min-lmwt 4 --max-lmwt 15" + +num_threads=1 # if >1, will use latgen-faster-parallel +parallel_opts= # Ignored now. +use_gpu="no" # yes|no|optionaly + +cmvn_opts= +splice_opts= +delta_opts= + +asr_propagator= +nnet_forward= +# End configuration section. + +echo "$0 $@" # Print the command line for logging + +[ -f ./path.sh ] && . ./path.sh; # source the path. +. parse_options.sh || exit 1; + +if [ $# != 4 ]; then + echo "Usage: $0 [options] " + echo "... where is assumed to be a sub-directory of the directory" + echo " where the DNN and transition model is." + echo "e.g.: $0 exp/dnn1/graph_tgpr data/test config.lua exp/dnn1/decode_tgpr" + echo "" + echo "This script works on plain or modified features (CMN,delta+delta-delta)," + echo "which are then sent through feature-transform. It works out what type" + echo "of features you used from content of srcdir." + echo "" + echo "main options (for others, see top of script file)" + echo " --config # config containing options" + echo " --nj # number of parallel jobs" + echo " --cmd (utils/run.pl|utils/queue.pl ) # how to run jobs." + echo "" + echo " --srcdir # non-default dir with DNN/models, can be different" + echo " # from parent dir of ' (opt.)" + echo "" + echo " --acwt # select acoustic scale for decoding" + echo " --scoring-opts # options forwarded to local/score.sh" + echo " --num-threads # N>1: run multi-threaded decoder" + exit 1; +fi + + +graphdir=$1 +data=$2 +model_conf=$3 +dir=$4 + +[ -z $srcdir ] && srcdir=`dirname $dir`; # Default model directory one level up from decoding directory. +sdata=$data/split$nj; + +mkdir -p $dir/log + +[[ -d $sdata && $data/feats.scp -ot $sdata ]] || split_data.sh $data $nj || exit 1; +echo $nj > $dir/num_jobs + +# Select default locations to model files (if not already set externally) +[ -z "$model" ] && model=$srcdir/final.mdl +# +[ -z "$class_frame_counts" -a -f $srcdir/prior_counts ] && class_frame_counts=$srcdir/prior_counts # priority, +[ -z "$class_frame_counts" ] && class_frame_counts=$srcdir/ali_train_pdf.counts + +# Check that files exist +for f in $sdata/1/feats.scp $model $class_frame_counts $graphdir/HCLG.fst; do + [ ! -f $f ] && echo "$0: missing file $f" && exit 1; +done + +# Possibly use multi-threaded decoder +thread_string= +[ $num_threads -gt 1 ] && thread_string="-parallel --num-threads=$num_threads" + + +# PREPARE FEATURE EXTRACTION PIPELINE +# import config, +D=$srcdir +[ -e $D/norm_vars ] && cmvn_opts="--norm-means=true --norm-vars=$(cat $D/norm_vars)" # Bwd-compatibility, +[ -e $D/cmvn_opts ] && cmvn_opts=$(cat $D/cmvn_opts) +[ -e $D/splice_opts ] && splice_opts=$(cat $D/splice_opts) +[ -e $D/delta_order ] && delta_opts="--delta-order=$(cat $D/delta_order)" # Bwd-compatibility, +[ -e $D/delta_opts ] && delta_opts=$(cat $D/delta_opts) +# +# Create the feature stream, +feats="ark,s,cs:copy-feats scp:$sdata/JOB/feats.scp ark:- |" +# apply-cmvn (optional), +[ ! -z "$cmvn_opts" -a ! -f $sdata/1/cmvn.scp ] && echo "$0: Missing $sdata/1/cmvn.scp" && exit 1 +[ ! -z "$cmvn_opts" ] && feats="$feats apply-cmvn $cmvn_opts --utt2spk=ark:$sdata/JOB/utt2spk scp:$sdata/JOB/cmvn.scp ark:- ark:- |" +# splice-opts (optional), +[ ! -z "$splice_opts" ] && feats="$feats splice-feats $splice_opts ark:- ark:- |" +# add-deltas (optional), +[ ! -z "$delta_opts" ] && feats="$feats add-deltas $delta_opts ark:- ark:- |" +# +# Run the decoding in the queue, +if [ $stage -le 0 ]; then +# $cmd --num-threads $((num_threads+1)) JOB=1:$nj $dir/log/decode.JOB.log \ +# remove multi-threads to avoid smp requirement + $cmd --num-threads $((num_threads)) JOB=1:$nj $dir/log/decode.JOB.log \ + $nnet_forward $nnet_forward_opts --class-frame-counts=$class_frame_counts --use-gpu=$use_gpu $model_conf "$feats" ark:- $asr_propagator \| \ + latgen-faster-mapped$thread_string --min-active=$min_active --max-active=$max_active --max-mem=$max_mem --beam=$beam \ + --lattice-beam=$lattice_beam --acoustic-scale=$acwt --allow-partial=true --word-symbol-table=$graphdir/words.txt \ + $model $graphdir/HCLG.fst ark:- "ark:|gzip -c > $dir/lat.JOB.gz" || exit 1; +fi +# Run the scoring +if ! $skip_scoring ; then + [ ! -x local/score.sh ] && \ + echo "Not scoring because local/score.sh does not exist or not executable." && exit 1; + local/score.sh $scoring_opts --cmd "$cmd" $data $graphdir $dir || exit 1; +fi + +exit 0; diff --git a/kaldi_decode/kaldi_decode-scm-1.rockspec b/kaldi_decode/kaldi_decode-scm-1.rockspec new file mode 100644 index 0000000..cc533ae --- /dev/null +++ b/kaldi_decode/kaldi_decode-scm-1.rockspec @@ -0,0 +1,36 @@ +package = "kaldi_decode" +version = "scm-1" +source = { + url = "https://github.com/Nerv-SJTU/nerv-speech.git" +} +description = { + summary = "Kaldi decode support for NERV", + detailed = [[ + ]], + homepage = "https://github.com/Determinant/nerv-speech", + license = "BSD" +} +dependencies = { + "nerv >= scm-1", + "lua >= 5.1" +} +build = { + type = "make", + build_variables = { + CFLAGS="$(CFLAGS) -Wall -Wextra -g -O2", + --CFLAGS="$(CFLAGS) -Wall -Wextra -g", + LIBFLAG="$(LIBFLAG)", + LUA_LIBDIR="$(LUA_LIBDIR)", + LUA_BINDIR="$(LUA_BINDIR)", + LUA_INCDIR="$(LUA_INCDIR)", + LUA="$(LUA)", + }, + install_variables = { + LUA_BINDIR="$(LUA_BINDIR)", + INST_PREFIX="$(PREFIX)", + INST_BINDIR="$(BINDIR)", + INST_LIBDIR="$(LIBDIR)", + INST_LUADIR="$(LUADIR)", + INST_CONFDIR="$(CONFDIR)", + }, +} diff --git a/kaldi_decode/local/score.sh b/kaldi_decode/local/score.sh deleted file mode 100755 index b18f350..0000000 --- a/kaldi_decode/local/score.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# Copyright 2012 Johns Hopkins University (Author: Daniel Povey) -# Apache 2.0 - -[ -f ./path.sh ] && . ./path.sh - -# begin configuration section. -cmd=run.pl -stage=0 -decode_mbr=true -reverse=false -word_ins_penalty=0.0 -min_lmwt=5 -max_lmwt=20 -#end configuration section. - -[ -f ./path.sh ] && . ./path.sh -. parse_options.sh || exit 1; - -if [ $# -ne 3 ]; then - echo "Usage: local/score.sh [--cmd (run.pl|queue.pl...)] " - echo " Options:" - echo " --cmd (run.pl|queue.pl...) # specify how to run the sub-processes." - echo " --stage (0|1|2) # start scoring script from part-way through." - echo " --decode_mbr (true/false) # maximum bayes risk decoding (confusion network)." - echo " --min_lmwt # minumum LM-weight for lattice rescoring " - echo " --max_lmwt # maximum LM-weight for lattice rescoring " - echo " --reverse (true/false) # score with time reversed features " - exit 1; -fi - -data=$1 -lang_or_graph=$2 -dir=$3 - -symtab=$lang_or_graph/words.txt - -for f in $symtab $dir/lat.1.gz $data/text; do - [ ! -f $f ] && echo "score.sh: no such file $f" && exit 1; -done - -mkdir -p $dir/scoring/log - -cat $data/text | sed 's:::g' | sed 's:::g' > $dir/scoring/test_filt.txt - -$cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/best_path.LMWT.log \ - lattice-scale --inv-acoustic-scale=LMWT "ark:gunzip -c $dir/lat.*.gz|" ark:- \| \ - lattice-add-penalty --word-ins-penalty=$word_ins_penalty ark:- ark:- \| \ - lattice-best-path --word-symbol-table=$symtab \ - ark:- ark,t:$dir/scoring/LMWT.tra || exit 1; - -if $reverse; then - for lmwt in `seq $min_lmwt $max_lmwt`; do - mv $dir/scoring/$lmwt.tra $dir/scoring/$lmwt.tra.orig - awk '{ printf("%s ",$1); for(i=NF; i>1; i--){ printf("%s ",$i); } printf("\n"); }' \ - <$dir/scoring/$lmwt.tra.orig >$dir/scoring/$lmwt.tra - done -fi - -# Note: the double level of quoting for the sed command -$cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/score.LMWT.log \ - cat $dir/scoring/LMWT.tra \| \ - utils/int2sym.pl -f 2- $symtab \| sed 's:\::g' \| \ - compute-wer --text --mode=present \ - ark:$dir/scoring/test_filt.txt ark,p:- ">&" $dir/wer_LMWT || exit 1; - -exit 0; diff --git a/kaldi_decode/nnet-forward-with-nerv.sh b/kaldi_decode/nnet-forward-with-nerv.sh new file mode 100644 index 0000000..71bf239 --- /dev/null +++ b/kaldi_decode/nnet-forward-with-nerv.sh @@ -0,0 +1,2 @@ +#!/bin/bash +source <(../../install/bin/luarocks path) diff --git a/kaldi_decode/path.sh b/kaldi_decode/path.sh deleted file mode 100755 index 5e9bd2a..0000000 --- a/kaldi_decode/path.sh +++ /dev/null @@ -1,11 +0,0 @@ -### change this line to your kaldi repo -export KALDI_ROOT=/speechlab/tools/KALDI/kaldi-master/ -### the following lines should not be changed in most cases - -# setup kaldi path -[ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh -export PATH=$PWD/utils/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/tools/irstlm/bin/:$KALDI_ROOT/src/fstbin/:$KALDI_ROOT/src/gmmbin/:$KALDI_ROOT/src/featbin/:$KALDI_ROOT/src/lm/:$KALDI_ROOT/src/sgmmbin/:$KALDI_ROOT/src/sgmm2bin/:$KALDI_ROOT/src/fgmmbin/:$KALDI_ROOT/src/latbin/:$KALDI_ROOT/src/nnetbin:$KALDI_ROOT/src/nnet2bin/:$KALDI_ROOT/src/kwsbin:$PWD:$PATH -export LC_ALL=C - -# setup luarocks path and cpath for NERV (important) -source <(../../install/bin/luarocks path) diff --git a/kaldi_decode/src/Makefile b/kaldi_decode/src/Makefile deleted file mode 100644 index 0897798..0000000 --- a/kaldi_decode/src/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# Change KDIR to `kaldi-trunk' path (Kaldi must be compiled with --share) -KDIR := /speechlab/tools/KALDI/kaldi-master/ -NERVDIR := /speechlab/users/mfy43/nerv/ -CUDADIR := /usr/local/cuda/ - -nnet-forward: - g++ -msse -msse2 -Wall -I $(KDIR)/src/ -pthread -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -Wno-sign-compare -Wno-unused-local-typedefs -Winit-self -DHAVE_EXECINFO_H=1 -rdynamic -DHAVE_CXXABI_H -DHAVE_ATLAS -I $(KDIR)/tools/ATLAS/include -I $(KDIR)/tools/openfst/include -Wno-sign-compare -g -fPIC -DHAVE_CUDA -I $(CUDADIR)/include -DKALDI_NO_EXPF -I $(NERVDIR)/install//include/luajit-2.0/ -I $(NERVDIR)/install/include/ -DLUA_USE_APICHECK -c -o nnet-forward.o nnet-forward.cc - g++ -rdynamic -Wl,-rpath=$(KDIR)/tools/openfst/lib -L$(CUDADIR)/lib64 -Wl,-rpath,$(CUDADIR)/lib64 -Wl,-rpath=$(KDIR)/src/lib -L. -L$(KDIR)/src/nnet/ -L$(KDIR)/src/cudamatrix/ -L$(KDIR)/src/lat/ -L$(KDIR)/src/hmm/ -L$(KDIR)/src/tree/ -L$(KDIR)/src/matrix/ -L$(KDIR)/src/util/ -L$(KDIR)/src/base/ nnet-forward.o $(KDIR)/src/nnet//libkaldi-nnet.so $(KDIR)/src/cudamatrix//libkaldi-cudamatrix.so $(KDIR)/src/lat//libkaldi-lat.so $(KDIR)/src/hmm//libkaldi-hmm.so $(KDIR)/src/tree//libkaldi-tree.so $(KDIR)/src/matrix//libkaldi-matrix.so $(KDIR)/src/util//libkaldi-util.so $(KDIR)/src/base//libkaldi-base.so -L$(KDIR)/tools/openfst/lib -lfst -lm -lpthread -ldl -lkaldi-nnet -lkaldi-cudamatrix -lkaldi-lat -lkaldi-hmm -lkaldi-tree -lkaldi-matrix -lkaldi-util -lkaldi-base -lstdc++ -L$(NERVDIR)/install/lib -Wl,-rpath=$(NERVDIR)/install/lib -lnervcore -lluaT -rdynamic -Wl,-rpath=$(KDIR)//tools/openfst/lib -L$(DUDADIR)/lib64 -Wl,-rpath,$(CUDADIR)/lib64 -Wl,-rpath=$(KDIR)//src/lib -lfst -lm -lpthread -ldl -L $(NERVDIR)/luajit-2.0/src/ -lluajit -o nnet-forward -L/home/intel/mkl/lib/intel64/ -Wl,-rpath=/home/intel/mkl/lib/intel64/ -lmkl_rt - -clean: - -rm nnet-forward.o nnet-forward - diff --git a/kaldi_decode/src/asr_propagator.lua b/kaldi_decode/src/asr_propagator.lua new file mode 100644 index 0000000..5d0ad7c --- /dev/null +++ b/kaldi_decode/src/asr_propagator.lua @@ -0,0 +1,84 @@ +print = function(...) io.write(table.concat({...}, "\t")) end +io.output('/dev/null') +-- path and cpath are correctly set by `path.sh` +local k,l,_=pcall(require,"luarocks.loader") _=k and l.add_context("nerv","scm-1") +require 'nerv' +nerv.printf("*** NERV: A Lua-based toolkit for high-performance deep learning (alpha) ***\n") +nerv.info("automatically initialize a default MContext...") +nerv.MMatrix._default_context = nerv.MContext() +nerv.info("the default MContext is ok") +-- only for backward compatibilty, will be removed in the future +local function _add_profile_method(cls) + local c = cls._default_context + cls.print_profile = function () c:print_profile() end + cls.clear_profile = function () c:clear_profile() end +end +_add_profile_method(nerv.MMatrix) + +function build_propagator(ifname, feature) + local param_repo = nerv.ParamRepo() + param_repo:import(ifname, nil, gconf) + local layer_repo = make_layer_repo(param_repo) + local network = get_decode_network(layer_repo) + local global_transf = get_global_transf(layer_repo) + local input_order = get_decode_input_order() + local readers = make_decode_readers(feature, layer_repo) + + local batch_propagator = function() + local data = nil + for ri = 1, #readers do + data = readers[ri].reader:get_data() + if data ~= nil then + break + end + end + + if data == nil then + return "", nil + end + + gconf.batch_size = data[input_order[1].id]:nrow() + network:init(gconf.batch_size) + + local input = {} + for i, e in ipairs(input_order) do + local id = e.id + if data[id] == nil then + nerv.error("input data %s not found", id) + end + local transformed + if e.global_transf then + transformed = nerv.speech_utils.global_transf(data[id], + global_transf, + gconf.frm_ext or 0, 0, + gconf) + else + transformed = data[id] + end + table.insert(input, transformed) + end + local output = {nerv.MMatrixFloat(input[1]:nrow(), network.dim_out[1])} + network:propagate(input, output) + + local utt = data["key"] + if utt == nil then + nerv.error("no key found.") + end + + collectgarbage("collect") + return utt, output[1] + end + + return batch_propagator +end + +function init(config, feature) + dofile(config) + gconf.use_cpu = true -- use CPU to decode + trainer = build_propagator(gconf.decode_param, feature) +end + +function feed() + local utt, mat = trainer() + return utt, mat +end diff --git a/kaldi_decode/src/nerv4decode.lua b/kaldi_decode/src/nerv4decode.lua deleted file mode 100644 index 898b5a8..0000000 --- a/kaldi_decode/src/nerv4decode.lua +++ /dev/null @@ -1,86 +0,0 @@ -print = function(...) io.write(table.concat({...}, "\t")) end -io.output('/dev/null') --- path and cpath are correctly set by `path.sh` -local k,l,_=pcall(require,"luarocks.loader") _=k and l.add_context("nerv","scm-1") -require 'nerv' -nerv.printf("*** NERV: A Lua-based toolkit for high-performance deep learning (alpha) ***\n") -nerv.info("automatically initialize a default MContext...") -nerv.MMatrix._default_context = nerv.MContext() -nerv.info("the default MContext is ok") --- only for backward compatibilty, will be removed in the future -local function _add_profile_method(cls) - local c = cls._default_context - cls.print_profile = function () c:print_profile() end - cls.clear_profile = function () c:clear_profile() end -end -_add_profile_method(nerv.MMatrix) - -function build_trainer(ifname, feature) - local param_repo = nerv.ParamRepo() - param_repo:import(ifname, nil, gconf) - local layer_repo = make_layer_repo(param_repo) - local network = get_decode_network(layer_repo) - local global_transf = get_global_transf(layer_repo) - local input_order = get_input_order() - local readers = make_readers(feature, layer_repo) - network:init(1) - - local iterative_trainer = function() - local data = nil - for ri = 1, #readers, 1 do - data = readers[ri].reader:get_data() - if data ~= nil then - break - end - end - - if data == nil then - return "", nil - end - - local input = {} - for i, e in ipairs(input_order) do - local id = e.id - if data[id] == nil then - nerv.error("input data %s not found", id) - end - local transformed - if e.global_transf then - transformed = nerv.speech_utils.global_transf(data[id], - global_transf, - gconf.frm_ext or 0, 0, - gconf) - else - transformed = data[id] - end - table.insert(input, transformed) - end - local output = {nerv.MMatrixFloat(input[1]:nrow(), network.dim_out[1])} - network:batch_resize(input[1]:nrow()) - network:propagate(input, output) - - local utt = data["key"] - if utt == nil then - nerv.error("no key found.") - end - - local mat = nerv.MMatrixFloat(output[1]:nrow(), output[1]:ncol()) - output[1]:copy_toh(mat) - - collectgarbage("collect") - return utt, mat - end - - return iterative_trainer -end - -function init(config, feature) - dofile(config) - gconf.use_cpu = true -- use CPU to decode - trainer = build_trainer(gconf.decode_param, feature) -end - -function feed() - local utt, mat = trainer() - return utt, mat -end diff --git a/kaldi_decode/src/nnet-forward.cc b/kaldi_decode/src/nnet-forward.cc index 4911791..8781705 100644 --- a/kaldi_decode/src/nnet-forward.cc +++ b/kaldi_decode/src/nnet-forward.cc @@ -46,7 +46,7 @@ int main(int argc, char *argv[]) { const char *usage = "Perform forward pass through Neural Network.\n" "\n" - "Usage: nnet-forward [options] [nerv4decode.lua]\n" + "Usage: nnet-forward [options] [asr_propagator.lua]\n" "e.g.: \n" " nnet-forward config.lua ark:features.ark ark:mlpoutput.ark\n"; @@ -78,9 +78,9 @@ int main(int argc, char *argv[]) { std::string config = po.GetArg(1), feature_rspecifier = po.GetArg(2), feature_wspecifier = po.GetArg(3), - nerv4decode = "src/nerv4decode.lua"; - if(po.NumArgs() >= 4) - nerv4decode = po.GetArg(4); + propagator = "src/asr_propagator.lua"; + if(po.NumArgs() >= 4) + propagator = po.GetArg(4); //Select the GPU #if HAVE_CUDA==1 @@ -99,8 +99,8 @@ int main(int argc, char *argv[]) { lua_State *L = lua_open(); luaL_openlibs(L); - if(luaL_loadfile(L, nerv4decode.c_str())) - KALDI_ERR << "luaL_loadfile() " << nerv4decode << " failed " << lua_tostring(L, -1); + if(luaL_loadfile(L, propagator.c_str())) + KALDI_ERR << "luaL_loadfile() " << propagator << " failed " << lua_tostring(L, -1); if(lua_pcall(L, 0, 0, 0)) KALDI_ERR << "lua_pall failed " << lua_tostring(L, -1); diff --git a/kaldi_io/Makefile b/kaldi_io/Makefile index 6d350a4..abfa8e6 100644 --- a/kaldi_io/Makefile +++ b/kaldi_io/Makefile @@ -1,6 +1,12 @@ -# Change KDIR to `kaldi-trunk' path (Kaldi must be compiled with --share) -KDIR := /speechlab/tools/KALDI/kaldi-master/ +ifndef LUA_BINDIR +$(error Please build the package via luarocks: `luarocks make`) +endif +ifndef KALDI_BASE +$(error KALDI_BASE is not set) +endif + +KDIR := $(KALDI_BASE) SHELL := /bin/bash BUILD_DIR := $(CURDIR)/build INC_PATH := $(LUA_BINDIR)/../include/ @@ -26,7 +32,6 @@ build: $(OBJ_DIR) $(OBJ_SUBDIR) $(OBJS) $(OBJ_DIR)/src/test install: $(LUA_DIR) $(LUA_SUBDIR) $(LUA_LIBS) $(LIBS) include $(KDIR)/src/kaldi.mk -#KL := $(KDIR)/src/feat/kaldi-feat.a $(KDIR)/src/matrix/kaldi-matrix.a $(KDIR)/src/base/kaldi-base.a $(KDIR)/src/util/kaldi-util.a $(KDIR)/src/hmm/kaldi-hmm.a $(KDIR)/src/tree/kaldi-tree.a -lcblas -llapack_atlas KL := $(KDIR)/src/feat/kaldi-feat.a $(KDIR)/src/matrix/kaldi-matrix.a $(KDIR)/src/base/kaldi-base.a $(KDIR)/src/util/kaldi-util.a $(KDIR)/src/hmm/kaldi-hmm.a $(KDIR)/src/tree/kaldi-tree.a $(BLAS_LDFLAGS) $(OBJ_DIR) $(LUA_DIR) $(OBJ_SUBDIR) $(LUA_SUBDIR): -mkdir -p $@ diff --git a/kaldi_io/kaldi_io-scm-1.rockspec b/kaldi_io/kaldi_io-scm-1.rockspec index 7c9f8d8..5a97cff 100644 --- a/kaldi_io/kaldi_io-scm-1.rockspec +++ b/kaldi_io/kaldi_io-scm-1.rockspec @@ -4,7 +4,7 @@ source = { url = "https://github.com/Nerv-SJTU/nerv-speech.git" } description = { - summary = "Kaldi I/O support (Kaldi I/O wrapper) for Nerv", + summary = "Kaldi I/O support (Kaldi I/O wrapper) for NERV", detailed = [[ ]], homepage = "https://github.com/Nerv-SJTU/nerv-speech", diff --git a/kaldi_io/tools/convert_from_kaldi_pretrain.sh b/kaldi_io/tools/convert_from_kaldi_pretrain.sh new file mode 100755 index 0000000..dc8ec8e --- /dev/null +++ b/kaldi_io/tools/convert_from_kaldi_pretrain.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +hid_dim=1024 +hid_num=6 +pretrain_dir=exp/dnn4_pretrain-dbn +nerv_kaldi=/speechlab/users/mfy43/nerv/speech/kaldi_io/ + +[ -f path.sh ] && . ./path.sh +. parse_options.sh || exit 1; + +data=$1 +data_cv=$2 +lang=$3 +alidir=$4 +alidir_cv=$5 +dir=$6 + +[[ -z $data_fmllr ]] && data_fmllr=data-fmllr-tri3 +[[ -z $alidir ]] && alidir=exp/tri3_ali +[[ -z $dir ]] && dir=exp/dnn4_nerv_prepare +[[ -z $data ]] && data=$data_fmllr/train_tr90 +[[ -z $data_cv ]] && data_cv=$data_fmllr/train_cv10 +kaldi_to_nerv=$nerv_kaldi/tools/kaldi_to_nerv +mkdir $dir -p +mkdir $dir/log -p +false && { +###### PREPARE DATASETS ###### +cp $data/feats.scp $dir/train_sorted.scp +cp $data_cv/feats.scp $dir/cv.scp +utils/shuffle_list.pl --srand ${seed:-777} <$dir/train_sorted.scp >$dir/train.scp + +feats_tr="ark:copy-feats scp:$dir/train.scp ark:- |" + +###### INITIALIZE OUTPUT LAYER ###### +[ -z $num_tgt ] && \ + num_tgt=$(hmm-info --print-args=false $alidir/final.mdl | grep pdfs | awk '{ print $NF }') +nnet_proto=$dir/nnet_output.proto +echo "# genrating network prototype $nnet_proto" +utils/nnet/make_nnet_proto.py \ + $hid_dim $num_tgt 0 $hid_dim >$nnet_proto || exit 1 +nnet_init=$dir/nnet_output.init +nnet-initialize --binary=false $nnet_proto $nnet_init + +###### MODEL PARAMETER CONVERSION ###### +$kaldi_to_nerv $nnet_init $dir/nnet_output.nerv +$kaldi_to_nerv <(nnet-copy --binary=false $pretrain_dir/${hid_num}.dbn -) $dir/nnet_init.nerv +$kaldi_to_nerv <(nnet-copy --binary=false $pretrain_dir/final.feature_transform -) $dir/nnet_trans.nerv +} +###### PREPARE FOR DECODING ##### +echo "Using PDF targets from dirs '$alidir' '$alidir_cv'" +# training targets in posterior format, +labels_tr="ark:ali-to-pdf $alidir/final.mdl \"ark:gunzip -c $alidir/ali.*.gz |\" ark:- | ali-to-post ark:- ark:- |" +labels_cv="ark:ali-to-pdf $alidir/final.mdl \"ark:gunzip -c $alidir_cv/ali.*.gz |\" ark:- | ali-to-post ark:- ark:- |" +# training targets for analyze-counts, +labels_tr_pdf="ark:ali-to-pdf $alidir/final.mdl \"ark:gunzip -c $alidir/ali.*.gz |\" ark:- |" +labels_tr_phn="ark:ali-to-phones --per-frame=true $alidir/final.mdl \"ark:gunzip -c $alidir/ali.*.gz |\" ark:- |" + +# get pdf-counts, used later for decoding/aligning, +analyze-counts --verbose=1 --binary=false "$labels_tr_pdf" $dir/ali_train_pdf.counts 2>$dir/log/analyze_counts_pdf.log || exit 1 +# copy the old transition model, will be needed by decoder, +copy-transition-model --binary=false $alidir/final.mdl $dir/final.mdl || exit 1 +# copy the tree +cp $alidir/tree $dir/tree || exit 1 + +# make phone counts for analysis, +[ -e $lang/phones.txt ] && analyze-counts --verbose=1 --symbol-table=$lang/phones.txt "$labels_tr_phn" /dev/null 2>$dir/log/analyze_counts_phones.log || exit 1 diff --git a/kaldi_io/tools/kaldi_to_nerv b/kaldi_io/tools/kaldi_to_nerv new file mode 100755 index 0000000..d08894d Binary files /dev/null and b/kaldi_io/tools/kaldi_to_nerv differ diff --git a/kaldi_seq/Makefile b/kaldi_seq/Makefile index e76eea8..c712319 100644 --- a/kaldi_seq/Makefile +++ b/kaldi_seq/Makefile @@ -1,6 +1,12 @@ -# Change KDIR to `kaldi-trunk' path (Kaldi must be compiled with --share) -KDIR := /slfs6/users/ymz09/kaldi/ +ifndef LUA_BINDIR +$(error Please build the package via luarocks: `luarocks make`) +endif +ifndef KALDI_BASE +$(error KALDI_BASE is not set) +endif + +KDIR := $(KALDI_BASE) SHELL := /bin/bash BUILD_DIR := $(CURDIR)/build INC_PATH := $(LUA_BINDIR)/../include/ diff --git a/tutorial/howto_pretrain_from_kaldi.rst b/tutorial/howto_pretrain_from_kaldi.rst new file mode 100644 index 0000000..95b5f36 --- /dev/null +++ b/tutorial/howto_pretrain_from_kaldi.rst @@ -0,0 +1,60 @@ +How to Use a Pretrained nnet Model from Kaldi +============================================= + +:author: Ted Yin (mfy43) +:abstract: Instruct on how to pretrain a basic dnn with timit dataset using + Kaldi and then convert the pretrained model to nerv format to let + NERV finetune. Finally it shows two possible ways to decode the + finetuned model in Kaldi framework. + +- Locate the egs/timit inside Kaldi trunk directory. + +- Configure ``cmd.sh`` and ``path.sh`` according to your machine setting. + +- Open the ``run.sh`` and locate the line saying ``exit 0 # From this point + you can run Karel's DNN: local/nnet/run_dnn.sh``. Uncomment this line. This + is because in this tutorial, we only want to train a basic tri-phone DNN, + so we simply don't do MMI training, system combination or fancy things like + these. + +- Run ``./run.sh`` to start the training stages. After that, we will get + tri-phone GMM-HMM trained and the aligned labels. Let's move forward to + pretrain a DNN. + +- Open ``local/nnet/run_dnn.sh``, there are again several stages. Note that + the first stage is what we actually need (pretraining the DNN), since in + this tutorial we want to demonstrate how to get the pretrained model from + stage 1, replace stage 2 with NERV (finetune per-frame cross-entropy), and + decode using the finetuned network. However, here we add a line ``exit 0`` + after stage 2 to preserve stage 2 in order to compare the NERV result + against the standard one (the decode result using finetuned model produced + by the original stage 2). + +- Run ``local/nnet/run_dnn.sh`` (first two stages). +- You'll find directory like ``dnn4_pretrain-dbn`` and ``dnn4_pretrain-dbn_dnn`` inside the ``exp/``. They correspond to stage 1 and stage 2 respectively. To use NERV to do stage 2 instead, we need the pretrained network and the global transformation from stage 1: + + - Check the file ``exp/dnn4_pretrain-dbn/6.dbn`` exists. (pretrained network) + - Check the file ``exp/dnn4_pretrain-dbn/tr_splice5_cmvn-g.nnet`` exists. (global transformation) + - Run script from ``kaldi_io/tools/convert_from_kaldi_pretrain.sh`` to generate the parameters for the output layer and the script files for training and cross-validation set. + + - The previous conversion commands will automatically give identifiers to the + parameters read from the Kaldi network file. The identifiers are like, for + example, ``affine0_ltp`` and ``bias0``. These names should correspond to + the identifiers used in the declaration of the network. Luckily, this + tutorial comes with a written network declaration at + ``nerv/examples/timit_baseline2.lua``. + +- Copy the file ``nerv/examples/timit_baseline2.lua`` to + ``timit_mybaseline.lua``, and change the line containing ``/speechlab`` to + your own setting. + +- Start the NERV training by ``install/bin/nerv nerv/examples/asr_trainer.lua timit_mybaseline.lua``. + + - ``install/bin/nerv`` is the program which sets up the NERV environment, + + - followed by an argument ``nerv/examples/asr_trainer.lua`` which is the script + you actually want to run (the general DNN training scheduler), + + - followed by an argument ``timit_mybaseline.lua`` to the scheduler, + specifying the network you want to train and some relevant settings, such + as where to find the initialized parameters and learning rate, etc. -- cgit v1.2.3-70-g09d2