diff options
59 files changed, 2008 insertions, 1196 deletions
diff --git a/.gitmodules b/.gitmodules index 9f556c5..2b346c4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,9 @@ [submodule "luajit-2.0"] path = luajit-2.0 - url = http://luajit.org/git/luajit-2.0.git + url = https://speechlab.sjtu.edu.cn/gitlab/nerv-dev/luajit.git [submodule "luarocks"] path = luarocks - url = https://github.com/keplerproject/luarocks.git + url = https://speechlab.sjtu.edu.cn/gitlab/nerv-dev/luarocks.git +[submodule "Penlight"] + path = Penlight + url = https://speechlab.sjtu.edu.cn/gitlab/nerv-dev/Penlight.git @@ -1,19 +1,42 @@ .PHONY: all clean install luajit luarocks speech +############## EDIT THESE LINES ##################### SHELL := /bin/bash PREFIX := $(CURDIR)/install/ -all: luajit luarocks install +#CUDA_BASE := /usr/local/cuda-7.0 +CUDA_BASE := /usr/local/cuda +BLAS_BASE := /usr/lib/ +BLAS_LDFLAGS := -L$(BLAS_BASE) -Wl,-rpath=$(BLAS_BASE) +BLAS_TYPE := atlas +KALDI_BASE := /speechlab/tools/KALDI/kaldi-master/ +####################################################### +MKL_LDFLAGS := -lmkl_rt +ATLAS_LDFLAGS := -lcblas -llapack_atlas +ifeq ($(BLAS_TYPE), mkl) +BLAS_LDFLAGS += $(MKL_LDFLAGS) +else ifeq ($(BLAS_TYPE), atlas) +BLAS_LDFLAGS += $(ATLAS_LDFLAGS) +else +$(error Invalid blas type) +endif +export CUDA_BASE +export KALDI_BASE +export BLAS_LDFLAGS + +.PHONY: nerv speech/speech_utils speech/htk_io speech/kaldi_io speech/kaldi_decode \ + nerv-clean speech/speech_utils-clean speech/htk_io-clean speech/kaldi_io-clean speech/kaldi_decode-clean \ + Penlight + +all: luajit luarocks Penlight nerv luajit: PREFIX=$(PREFIX) ./tools/build_luajit.sh luarocks: PREFIX=$(PREFIX) ./tools/build_luarocks.sh -install: - cd nerv; $(PREFIX)/bin/luarocks make CFLAGS=$(CFLAGS) -speech: - cd speech/speech_utils; $(PREFIX)/bin/luarocks make - cd speech/htk_io; $(PREFIX)/bin/luarocks make - cd speech/kaldi_io; $(PREFIX)/bin/luarocks make -clean: - cd nerv && make clean - cd speech/speech_utils && make clean - cd speech/htk_io && make clean - cd speech/kaldi_io && make clean + +speech: speech/speech_utils speech/htk_io speech/kaldi_io speech/kaldi_decode +speech-clean: speech/speech_utils-clean speech/htk_io-clean speech/kaldi_io-clean speech/kaldi_decode-clean +clean: nerv-clean speech-clean + +nerv Penlight speech/speech_utils speech/htk_io speech/kaldi_io speech/kaldi_decode: + cd $@; $(PREFIX)/bin/luarocks make +nerv-clean speech/speech_utils-clean speech/htk_io-clean speech/kaldi_io-clean speech/kaldi_decode-clean: + cd $(subst -clean,,$@); make clean LUA_BINDIR=$(PREFIX)/bin/ diff --git a/Penlight b/Penlight new file mode 160000 +Subproject 16d149338af9efc910528641c5240c5641aeb8d diff --git a/README.md b/README.md deleted file mode 100644 index fe9dfc1..0000000 --- a/README.md +++ /dev/null @@ -1,55 +0,0 @@ -#The Nerv Toolkit User Manual# -NOTE: This readme is obsolete and will be rearranged, for further information, please check http://nerv-sjtu.github.io/nerv/ - -This user manual will information about how to use __Nerv__ and __Nerv__'s interface. - -##How to make and start using## -First make sure you have __lua__ and __CUDA__ installed on your computer. -__Nerv__ is currently developed via github.You can download and make __Nerv__ by doing the following: -``` -cd ~ -git clone https://github.com/Nerv-SJTU/nerv.git -cd nerv -git submodule init && git submodule update -make -#To include some new CUDA feature(e.x. atomicCAS), use "make CFLAGS=-D__NERV_FUTURE_CUDA_7" - -#further, if you want the speech modules -git clone https://github.com/Nerv-SJTU/nerv-speech.git speech -make speech -``` -The `git submodule` command is for the __luajit__ repository inside __Nerv__. -Now, you can try to run some example scripts. -``` -./install/bin/nerv examples/cumatrix_example.lua -``` -To get an example of DNN(for ASR) training, run(this requires the speech modules) -You need to be at or (copy files from) `/slfs1`(SJTU speechlab cluster) to get this running. -``` -./install/bin/nerv nerv/examples/asr_trainer.lua nerv/examples/swb_baseline.lua -``` - -##How to contribute## -Fork the original repository, then use the __pull&merge__ function in github to contribute. -The pull&merge request can be found on your dashboard in github. See this [sync-help] to sync with the original repository. - -##Nerv Packages## -* __luaT__ -Nerv uses [luaT]\(a [Torch] library\) to define lua class in C. -* __[The Nerv OOP](nerv/doc/nerv_class.md)__ -Enables object-oriented programming in Nerv. -* __[The Nerv utility functions](nerv/doc/nerv.md)__ -Inlcudes some utility functions from luaT to implement __Nerv.Class__. -* __[The Nerv Matrix Package](nerv/doc/nerv_matrix.md)__ -The matrix package is a basic package in __Nerv__ that is used to store and manipulate matrices. -* __[The Nerv IO Package](nerv/doc/nerv_io.md)__ -The IO package is used to read and write parameters to file. -* __[The Nerv Parameter Package](nerv/doc/nerv_param.md)__ -The parameter package is used to store, read model parameters from file. -* __[The Nerv Layer Package](nerv/doc/nerv_layer.md)__ -The layer package is used to define propagation and backpropagation of different type of layers. -* __[The Nerv NN Package](nerv/doc/nerv_nn.md)__ -The nn package is for organizing a neural network, it contains __nerv.LayerRepo__, __nerv.ParamRepo__, and __nerv.DAGLayer__. -[luaT]:https://github.com/torch/torch7/tree/master/lib/luaT -[Torch]:https://github.com/torch -[sync-help]:https://help.github.com/articles/syncing-a-fork/ diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..c00743c --- /dev/null +++ b/README.rst @@ -0,0 +1,64 @@ +NERV Toolkit +============ + +NOTE: This readme is in-progress. + +Installation +------------ +First, make sure you have at least one implementation of BLAS and CUDA installed +on your computer. + +- Checkout NERV: + + :: + + bash + git clone https://speechlab.sjtu.edu.cn/gitlab/nerv-dev/nerv.git + +- Checkout submodules (luajit, luarocks, Penlight, etc.): + + :: + + cd nerv + git submodule init && git submodule update + +- Build NERV: you can specify either ``mkl`` or ``atlas`` to ``BLAS_TYPE``. + ``BLAS_BASE`` is the directory containing BLAS ``.so`` files. By default, + ``atlas`` is used for ``BLAS_TYPE``, ``/usr/lib/`` is used for ``BLAS_BASE``, + and ``/usr/local/cuda`` is used for ``CUDA_BASE``. + + :: + + # an example for compiling on SJTU Speechlab major cluster + make BLAS_TYPE=mkl BLAS_BASE=/home/intel/mkl/lib/intel64/ CUDA_BASE=/usr/local/cuda + +- To include some new features (e.g. ``atomicCAS`` in CUDA), add corresponding flags to + ``NERV_FEAT`` (e.g. ``NERV_FEAT=-D__NERV_FUTURE_CUDA_7``) while making: + + :: + + make NERV_FEAT=-D__NERV_FUTURE_CUDA_7 BLAS_TYPE=mkl BLAS_BASE=/home/intel/mkl/lib/intel64/ CUDA_BASE=/usr/local/cuda + +- For speech tasks, you need to install related lua rocks (Lua packages): + + :: + + # checkout speech repository to local directory nerv/speech (suppose you're + # still at the root directory of NERV repo) + git clone https://speechlab.sjtu.edu.cn/gitlab/nerv-dev/nerv-speech.git speech + # build and install HTK I/O support, Kaldi I/O support, Kaldi decoding support, etc. + make speech BLAS_TYPE=mkl BLAS_BASE=/home/intel/mkl/lib/intel64/ + +Example & Tutorial +------------------ +For speech tasks, please refer to ``tutorial/`` in ``nerv-speech`` repository. + +Contribution +------------ +The basic rule is simple: just fork the original repository, then create a pull +request (merge request) to the administrator of the project. If you want to fix +any bugs in existing code, don't hesitate to create a pull (merge) request to +the repository with clear and detailed analysis of the problem. If you want to +add additional task-specific functionalities (modules) for speech to NERV, +please create a luarocks-compliant package and also a pull (merge) request to +the ``nerv-speech`` repository instead of ``nerv``. diff --git a/lua/config.lua b/lua/config.lua deleted file mode 100644 index 1ec1198..0000000 --- a/lua/config.lua +++ /dev/null @@ -1,67 +0,0 @@ -function get_global_conf() - local global_conf = { - lrate = 0.15, - wcost = 1e-5, - momentum = 0, - clip = 5, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - vocab_size = 10000, - nn_act_default = 0, - hidden_size = 300, - layer_num = 1, - chunk_size = 15, - batch_size = 20, - max_iter = 3, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout = 0.5, - timer = nerv.Timer(), - pr = nerv.ParamRepo(), - } - return global_conf -end - -function get_layers(global_conf) - local pr = global_conf.pr - local layers = { - ['nerv.LSTMLayer'] = {}, - ['nerv.DropoutLayer'] = {}, - ['nerv.SelectLinearLayer'] = { - ['select'] = {dim_in = {1}, dim_out = {global_conf.hidden_size}, vocab = global_conf.vocab_size, pr = pr}, - }, - ['nerv.CombinerLayer'] = {}, - ['nerv.AffineLayer'] = { - output = {dim_in = {global_conf.hidden_size}, dim_out = {global_conf.vocab_size}, pr = pr} - }, - ['nerv.SoftmaxCELayer'] = { - softmax = {dim_in = {global_conf.vocab_size, global_conf.vocab_size}, dim_out = {1}, compressed = true}, - }, - } - for i = 1, global_conf.layer_num do - layers['nerv.LSTMLayer']['lstm' .. i] = {dim_in = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, dim_out = {global_conf.hidden_size, global_conf.hidden_size}, pr = pr} - layers['nerv.DropoutLayer']['dropout' .. i] = {dim_in = {global_conf.hidden_size}, dim_out = {global_conf.hidden_size}} - layers['nerv.CombinerLayer']['dup' .. i] = {dim_in = {global_conf.hidden_size}, dim_out = {global_conf.hidden_size, global_conf.hidden_size}, lambda = {1}} - end - return layers -end - -function get_connections(global_conf) - local connections = { - {'<input>[1]', 'select[1]', 0}, - {'select[1]', 'lstm1[1]', 0}, - {'dropout' .. global_conf.layer_num .. '[1]', 'output[1]', 0}, - {'output[1]', 'softmax[1]', 0}, - {'<input>[2]', 'softmax[2]', 0}, - {'softmax[1]', '<output>[1]', 0}, - } - for i = 1, global_conf.layer_num do - table.insert(connections, {'lstm' .. i .. '[1]', 'dup' .. i .. '[1]', 0}) - table.insert(connections, {'lstm' .. i .. '[2]', 'lstm' .. i .. '[3]', 1}) - table.insert(connections, {'dup' .. i .. '[1]', 'lstm' .. i .. '[2]', 1}) - table.insert(connections, {'dup' .. i .. '[2]', 'dropout' .. i .. '[1]', 0}) - if i > 1 then - table.insert(connections, {'dropout' .. (i - 1) .. '[1]', 'lstm' .. i .. '[1]', 0}) - end - end - return connections -end diff --git a/lua/main.lua b/lua/main.lua deleted file mode 100644 index ce0270a..0000000 --- a/lua/main.lua +++ /dev/null @@ -1,45 +0,0 @@ -nerv.include('reader.lua') -nerv.include('timer.lua') -nerv.include('config.lua') -nerv.include(arg[1]) - -local global_conf = get_global_conf() -local timer = global_conf.timer - -timer:tic('IO') - -local data_path = 'nerv/nerv/examples/lmptb/PTBdata/' -local train_reader = nerv.Reader(data_path .. 'vocab', data_path .. 'ptb.valid.txt.adds') -local val_reader = nerv.Reader(data_path .. 'vocab', data_path .. 'ptb.valid.txt.adds') - -local train_data = train_reader:get_all_batch(global_conf) -local val_data = val_reader:get_all_batch(global_conf) - -local layers = get_layers(global_conf) -local connections = get_connections(global_conf) - -local NN = nerv.NN(global_conf, train_data, val_data, layers, connections) - -timer:toc('IO') -timer:check('IO') -io.flush() - -timer:tic('global') -local best_cv = 1e10 -for i = 1, global_conf.max_iter do - timer:tic('Epoch' .. i) - local train_ppl, val_ppl = NN:epoch() - if val_ppl < best_cv then - best_cv = val_ppl - else - global_conf.lrate = global_conf.lrate / 2.0 - end - nerv.printf('Epoch %d: %f %f %f\n', i, global_conf.lrate, train_ppl, val_ppl) - timer:toc('Epoch' .. i) - timer:check('Epoch' .. i) - io.flush() -end -timer:toc('global') -timer:check('global') -timer:check('network') -timer:check('gc') diff --git a/lua/network.lua b/lua/network.lua deleted file mode 100644 index 0c11321..0000000 --- a/lua/network.lua +++ /dev/null @@ -1,113 +0,0 @@ -nerv.include('select_linear.lua') - -local nn = nerv.class('nerv.NN') - -function nn:__init(global_conf, train_data, val_data, layers, connections) - self.gconf = global_conf - self.network = self:get_network(layers, connections) - self.train_data = self:get_data(train_data) - self.val_data = self:get_data(val_data) -end - -function nn:get_network(layers, connections) - self.gconf.dropout_rate = 0 - local layer_repo = nerv.LayerRepo(layers, self.gconf.pr, self.gconf) - local graph = nerv.GraphLayer('graph', self.gconf, - {dim_in = {1, self.gconf.vocab_size}, dim_out = {1}, - layer_repo = layer_repo, connections = connections}) - local network = nerv.Network('network', self.gconf, - {network = graph, clip = self.gconf.clip}) - network:init(self.gconf.batch_size, self.gconf.chunk_size) - return network -end - -function nn:get_data(data) - local err_output = {} - local softmax_output = {} - local output = {} - for i = 1, self.gconf.chunk_size do - err_output[i] = self.gconf.cumat_type(self.gconf.batch_size, 1) - softmax_output[i] = self.gconf.cumat_type(self.gconf.batch_size, self.gconf.vocab_size) - output[i] = self.gconf.cumat_type(self.gconf.batch_size, 1) - end - local ret = {} - for i = 1, #data do - ret[i] = {} - ret[i].input = {} - ret[i].output = {} - ret[i].err_input = {} - ret[i].err_output = {} - for t = 1, self.gconf.chunk_size do - ret[i].input[t] = {} - ret[i].output[t] = {} - ret[i].err_input[t] = {} - ret[i].err_output[t] = {} - ret[i].input[t][1] = data[i].input[t] - ret[i].input[t][2] = data[i].output[t] - ret[i].output[t][1] = output[t] - local err_input = self.gconf.mmat_type(self.gconf.batch_size, 1) - for j = 1, self.gconf.batch_size do - if t <= data[i].seq_len[j] then - err_input[j - 1][0] = 1 - else - err_input[j - 1][0] = 0 - end - end - ret[i].err_input[t][1] = self.gconf.cumat_type.new_from_host(err_input) - ret[i].err_output[t][1] = err_output[t] - ret[i].err_output[t][2] = softmax_output[t] - end - ret[i].seq_length = data[i].seq_len - ret[i].new_seq = {} - for j = 1, self.gconf.batch_size do - if data[i].seq_start[j] then - table.insert(ret[i].new_seq, j) - end - end - end - return ret -end - -function nn:process(data, do_train) - local timer = self.gconf.timer - local total_err = 0 - local total_frame = 0 - for id = 1, #data do - if do_train then - self.gconf.dropout_rate = self.gconf.dropout - data[id].do_train = true - else - self.gconf.dropout_rate = 0 - data[id].do_train = false - end - timer:tic('network') - self.network:mini_batch_init(data[id]) - self.network:propagate() - timer:toc('network') - for t = 1, self.gconf.chunk_size do - local tmp = data[id].output[t][1]:new_to_host() - for i = 1, self.gconf.batch_size do - if t <= data[id].seq_length[i] then - total_err = total_err + math.log10(math.exp(tmp[i - 1][0])) - total_frame = total_frame + 1 - end - end - end - if do_train then - timer:tic('network') - self.network:back_propagate() - self.network:update() - timer:toc('network') - end - timer:tic('gc') - collectgarbage('collect') - timer:toc('gc') - end - return math.pow(10, - total_err / total_frame) -end - -function nn:epoch() - local train_error = self:process(self.train_data, true) - local val_error = self:process(self.val_data, false) - return train_error, val_error -end diff --git a/lua/reader.lua b/lua/reader.lua deleted file mode 100644 index 0c7bcb6..0000000 --- a/lua/reader.lua +++ /dev/null @@ -1,113 +0,0 @@ -local Reader = nerv.class('nerv.Reader') - -function Reader:__init(vocab_file, input_file) - self:get_vocab(vocab_file) - self:get_seq(input_file) -end - -function Reader:get_vocab(vocab_file) - local f = io.open(vocab_file, 'r') - local id = 0 - self.vocab = {} - while true do - local word = f:read() - if word == nil then - break - end - self.vocab[word] = id - id = id + 1 - end - self.size = id -end - -function Reader:split(s, t) - local ret = {} - for x in (s .. t):gmatch('(.-)' .. t) do - table.insert(ret, x) - end - return ret -end - -function Reader:get_seq(input_file) - local f = io.open(input_file, 'r') - self.seq = {} - while true do - local seq = f:read() - if seq == nil then - break - end - seq = self:split(seq, ' ') - local tmp = {} - for i = 1, #seq do - if seq[i] ~= '' then - table.insert(tmp, self.vocab[seq[i]]) - end - end - table.insert(self.seq, tmp) - end -end - -function Reader:get_in_out(id, pos) - return self.seq[id][pos], self.seq[id][pos + 1], pos + 1 == #self.seq[id] -end - -function Reader:get_all_batch(global_conf) - local data = {} - local pos = {} - local offset = 1 - for i = 1, global_conf.batch_size do - pos[i] = nil - end - --while true do - for i = 1, 100 do - local input = {} - local output = {} - for i = 1, global_conf.chunk_size do - input[i] = global_conf.mmat_type(global_conf.batch_size, 1) - input[i]:fill(global_conf.nn_act_default) - output[i] = global_conf.mmat_type(global_conf.batch_size, 1) - output[i]:fill(global_conf.nn_act_default) - end - local seq_start = {} - local seq_end = {} - local seq_len = {} - for i = 1, global_conf.batch_size do - seq_start[i] = false - seq_end[i] = false - seq_len[i] = 0 - end - local has_new = false - for i = 1, global_conf.batch_size do - if pos[i] == nil then - if offset < #self.seq then</ |