From cce6efcdfbe50a59e260cb5d55ae2c77326dc67c Mon Sep 17 00:00:00 2001 From: txh18 Date: Fri, 4 Dec 2015 17:53:31 +0800 Subject: added testout command for lstmlm --- nerv/examples/lmptb/lm_trainer.lua | 15 +++++++++-- nerv/examples/lmptb/lstmlm_ptb_main.lua | 44 ++++++++++++++++++++++++++------- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/nerv/examples/lmptb/lm_trainer.lua b/nerv/examples/lmptb/lm_trainer.lua index 9ef4794..58d5bfc 100644 --- a/nerv/examples/lmptb/lm_trainer.lua +++ b/nerv/examples/lmptb/lm_trainer.lua @@ -17,8 +17,19 @@ function nerv.BiasParam:update_by_gradient(gradient) end --Returns: LMResult -function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train) - local reader = nerv.LMSeqReader(global_conf, global_conf.batch_size, global_conf.chunk_size, global_conf.vocab) +function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) + if p_conf == nil then + p_conf = {} + end + local reader + if p_conf.one_sen_report == true then --report log prob one by one sentence + if do_train == true then + nerv.warning("LMTrainer.lm_process_file_rnn: warning, one_sen_report is true while do_train is also true, strange") + end + reader = nerv.LMSeqReader(global_conf, 1, global_conf.max_sen_len, global_conf.vocab) + else + reader = nerv.LMSeqReader(global_conf, global_conf.batch_size, global_conf.chunk_size, global_conf.vocab) + end reader:open_file(fn) local result = nerv.LMResult(global_conf, global_conf.vocab) result:init("rnn") diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index 681c308..9f02324 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -195,20 +195,23 @@ local set = arg[1] --"test" if (set == "ptb") then -data_dir = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata' +root_dir = '/home/slhome/txh18/workspace' +data_dir = root_dir .. '/ptb/DATA' train_fn = data_dir .. '/ptb.train.txt.adds' valid_fn = data_dir .. '/ptb.valid.txt.adds' test_fn = data_dir .. '/ptb.test.txt.adds' vocab_fn = data_dir .. '/vocab' +qdata_dir = root_dir .. '/ptb/questionGen/gen' + global_conf = { - lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 2, + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, cumat_type = nerv.CuMatrixFloat, mmat_type = nerv.MMatrixFloat, nn_act_default = 0, - hidden_size = 650, - layer_num = 2, + hidden_size = 300, + layer_num = 1, chunk_size = 15, batch_size = 20, max_iter = 45, @@ -221,10 +224,11 @@ global_conf = { valid_fn = valid_fn, test_fn = test_fn, vocab_fn = vocab_fn, + max_sen_len = 90, sche_log_pre = "[SCHEDULER]:", log_w_num = 40000, --give a message when log_w_num words have been processed timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/nerv/play/ptbEXP/tnn_lstm_test' + work_dir_base = '/home/slhome/txh18/workspace/ptb/EXP-nerv/lstmlm_v1.0' } elseif (set == "msr_sc") then @@ -303,6 +307,9 @@ local commands_str = "train:test" local commands = {} local test_iter = -1 +--for testout(question) +local q_file = "ptb.test.txt.q10rs1_Msss.adds" + if arg[2] ~= nil then nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) loadstring(arg[2])() @@ -311,21 +318,22 @@ else nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) end -global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num --.. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' global_conf.param_fn = global_conf.work_dir .. "/params" global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) -global_conf.log_fn = global_conf.work_dir .. '/lstm_tnn_' .. commands_str .. '_log' +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%X_%m_%d",os.time()) commands = nerv.SUtil.parse_commands_set(commands_str) -nerv.printf("%s creating work_dir...\n", global_conf.sche_log_pre) -nerv.LMUtil.wait(1) +nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) +nerv.LMUtil.wait(2) os.execute("mkdir -p "..global_conf.work_dir) os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) --redirecting log outputs! nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) ----------------printing options--------------------------------- nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) @@ -441,3 +449,21 @@ if commands["test"] == 1 then global_conf.dropout_rate = 0 LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! end --if commands["test"] + +if commands["testout"] == 1 then + nerv.printf("===TEST OUT===\n") + nerv.printf("q_file:\t%s\n", q_file) + local q_fn = qdata_dir .. q_file + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false) --false update! +end --if commands["testout"] + + -- cgit v1.2.3-70-g09d2 From 6b674ff54cf61dbf583032455ed2c4c33dd0443c Mon Sep 17 00:00:00 2001 From: txh18 Date: Fri, 4 Dec 2015 18:01:18 +0800 Subject: ... --- nerv/examples/lmptb/lstmlm_ptb_main.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index 9f02324..08e4849 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -318,7 +318,7 @@ else nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) end -global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' global_conf.param_fn = global_conf.work_dir .. "/params" -- cgit v1.2.3-70-g09d2 From 7ee8988f21075246106a4d990190d0ef25fa82a8 Mon Sep 17 00:00:00 2001 From: txh18 Date: Fri, 4 Dec 2015 21:03:06 +0800 Subject: added one_sen_report to lm_process_file --- nerv/examples/lmptb/lm_trainer.lua | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/nerv/examples/lmptb/lm_trainer.lua b/nerv/examples/lmptb/lm_trainer.lua index 58d5bfc..3c7078e 100644 --- a/nerv/examples/lmptb/lm_trainer.lua +++ b/nerv/examples/lmptb/lm_trainer.lua @@ -22,15 +22,23 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) p_conf = {} end local reader + local chunk_size, batch_size if p_conf.one_sen_report == true then --report log prob one by one sentence if do_train == true then nerv.warning("LMTrainer.lm_process_file_rnn: warning, one_sen_report is true while do_train is also true, strange") end - reader = nerv.LMSeqReader(global_conf, 1, global_conf.max_sen_len, global_conf.vocab) + nerv.printf("lm_process_file_rnn: one_sen report mode, set batch_size to 1 and chunk_size to max_sen_len(%d)\n", + global_conf.max_sen_len) + batch_size = 1 + chunk_size = global_conf.max_sen_len else - reader = nerv.LMSeqReader(global_conf, global_conf.batch_size, global_conf.chunk_size, global_conf.vocab) + batch_size = global_conf.batch_size + chunk_size = global_conf.chunk_size end + + reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, global_conf.vocab) reader:open_file(fn) + local result = nerv.LMResult(global_conf, global_conf.vocab) result:init("rnn") if global_conf.dropout_rate ~= nil then @@ -41,7 +49,7 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) tnn:flush_all() --caution: will also flush the inputs from the reader! local next_log_wcn = global_conf.log_w_num - local neto_bakm = global_conf.mmat_type(global_conf.batch_size, 1) --space backup matrix for network output + local neto_bakm = global_conf.mmat_type(batch_size, 1) --space backup matrix for network output while (1) do global_conf.timer:tic('most_out_loop_lmprocessfile') @@ -53,9 +61,9 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) break end - for t = 1, global_conf.chunk_size do + for t = 1, chunk_size do tnn.err_inputs_m[t][1]:fill(1) - for i = 1, global_conf.batch_size do + for i = 1, batch_size do if bit.band(feeds.flags_now[t][i], nerv.TNN.FC.HAS_LABEL) == 0 then tnn.err_inputs_m[t][1][i - 1][0] = 0 end @@ -81,15 +89,26 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) end global_conf.timer:tic('tnn_afterprocess') - for t = 1, global_conf.chunk_size, 1 do + local sen_logp = {} + for t = 1, chunk_size, 1 do tnn.outputs_m[t][1]:copy_toh(neto_bakm) - for i = 1, global_conf.batch_size, 1 do + for i = 1, batch_size, 1 do if (feeds.labels_s[t][i] ~= global_conf.vocab.null_token) then --result:add("rnn", feeds.labels_s[t][i], math.exp(tnn.outputs_m[t][1][i - 1][0])) result:add("rnn", feeds.labels_s[t][i], math.exp(neto_bakm[i - 1][0])) + if sen_logp[i] == nil then + sen_logp[i] = 0 + end + sen_logp[i] = sen_logp[i] + neto_bakm[i - 1][0] end end end + if p_conf.one_sen_report == true then + for i = 1, batch_size do + nerv.printf("LMTrainer.lm_process_file_rnn: one_sen_report, %f\n", sen_logp[i]) + end + end + tnn:move_right_to_nextmb({0}) --only copy for time 0 global_conf.timer:toc('tnn_afterprocess') @@ -113,7 +132,6 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) end ]]-- - collectgarbage("collect") --break --debug -- cgit v1.2.3-70-g09d2 From 0dbc40a998e2879f27110bd449f4cae9550c4e41 Mon Sep 17 00:00:00 2001 From: txh18 Date: Fri, 4 Dec 2015 21:44:06 +0800 Subject: trying to use lstm_t_v2 for ptb --- nerv/examples/lmptb/lmptb/lstm_t_v2.lua | 123 ++++++++ nerv/examples/lmptb/lstmlm_ptb_main.lua | 10 +- nerv/examples/lmptb/lstmlm_v2_ptb_main.lua | 470 +++++++++++++++++++++++++++++ nerv/layer/gate_fff.lua | 56 ++-- nerv/tnn/layersT/lstm_t.lua | 2 +- 5 files changed, 628 insertions(+), 33 deletions(-) create mode 100644 nerv/examples/lmptb/lmptb/lstm_t_v2.lua create mode 100644 nerv/examples/lmptb/lstmlm_v2_ptb_main.lua diff --git a/nerv/examples/lmptb/lmptb/lstm_t_v2.lua b/nerv/examples/lmptb/lmptb/lstm_t_v2.lua new file mode 100644 index 0000000..d3fc920 --- /dev/null +++ b/nerv/examples/lmptb/lmptb/lstm_t_v2.lua @@ -0,0 +1,123 @@ +local LSTMLayerT = nerv.class('nerv.LSTMLayerTv2', 'nerv.LayerT') +--a version of LSTM that only feed h into the gates + +function LSTMLayerT:__init(id, global_conf, layer_conf) + --input1:x input2:h input3:c + self.id = id + self.dim_in = layer_conf.dim_in + self.dim_out = layer_conf.dim_out + self.gconf = global_conf + + --prepare a DAGLayerT to hold the lstm structure + local pr = layer_conf.pr + if pr == nil then + pr = nerv.ParamRepo() + end + + local function ap(str) + return self.id .. '.' .. str + end + + local layers = { + ["nerv.CombinerLayer"] = { + [ap("inputXDup")] = {{}, {["dim_in"] = {self.dim_in[1]}, + ["dim_out"] = {self.dim_in[1], self.dim_in[1], self.dim_in[1], self.dim_in[1]}, ["lambda"] = {1}}}, + [ap("inputHDup")] = {{}, {["dim_in"] = {self.dim_in[2]}, + ["dim_out"] = {self.dim_in[2], self.dim_in[2], self.dim_in[2], self.dim_in[2]}, ["lambda"] = {1}}}, + [ap("inputCDup")] = {{}, {["dim_in"] = {self.dim_in[3]}, + ["dim_out"] = {self.dim_in[3]}, ["lambda"] = {1}}}, + [ap("mainCDup")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, + ["dim_out"] = {self.dim_in[3], self.dim_in[3]}, ["lambda"] = {1, 1}}}, + }, + ["nerv.AffineLayer"] = { + [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2], self.dim_in[3]}, + ["dim_out"] = {self.dim_out[1]}, ["pr"] = pr}}, + }, + ["nerv.TanhLayer"] = { + [ap("mainTanhL")] = {{}, {["dim_in"] = {self.dim_out[1]}, ["dim_out"] = {self.dim_out[1]}}}, + [ap("outputTanhL")] = {{}, {["dim_in"] = {self.dim_out[1]}, ["dim_out"] = {self.dim_out[1]}}}, + }, + ["nerv.GateFLayer"] = { + [ap("forgetGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, + ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}}, + [ap("inputGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, + ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}}, + [ap("outputGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, + ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}}, + + }, + ["nerv.ElemMulLayer"] = { + [ap("inputGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}}, + [ap("forgetGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}}, + [ap("outputGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}}, + }, + } + + local layerRepo = nerv.LayerRepo(layers, pr, global_conf) + + local connections_t = { + ["[1]"] = ap("inputXDup[1]"), + ["[2]"] = ap("inputHDup[1]"), + ["[3]"] = ap("inputCDup[1]"), + + [ap("inputXDup[1]")] = ap("mainAffineL[1]"), + [ap("inputHDup[1]")] = ap("mainAffineL[2]"), + + [ap("mainAffineL[1]")] = ap("mainTanhL[1]"), + + [ap("inputXDup[2]")] = ap("inputGateL[1]"), + [ap("inputHDup[2]")] = ap("inputGateL[2]"), + + [ap("inputXDup[3]")] = ap("forgetGateL[1]"), + [ap("inputHDup[3]")] = ap("forgetGateL[2]"), + + [ap("mainTanhL[1]")] = ap("inputGMulL[1]"), + [ap("inputGateL[1]")] = ap("inputGMulL[2]"), + + [ap("inputCDup[1]")] = ap("forgetGMulL[1]"), + [ap("forgetGateL[1]")] = ap("forgetGMulL[2]"), + + [ap("inputGMulL[1]")] = ap("mainCDup[1]"), + [ap("forgetGMulL[1]")] = ap("mainCDup[2]"), + + [ap("inputXDup[4]")] = ap("outputGateL[1]"), + [ap("inputHDup[4]")] = ap("outputGateL[2]"), + + [ap("mainCDup[2]")] = "[2]", + [ap("mainCDup[1]")] = ap("outputTanhL[1]"), + + [ap("outputTanhL[1]")] = ap("outputGMulL[1]"), + [ap("outputGateL[1]")] = ap("outputGMulL[2]"), + + [ap("outputGMulL[1]")] = "[1]", + } + self.dagL = nerv.DAGLayerT(self.id, global_conf, + {["dim_in"] = self.dim_in, ["dim_out"] = self.dim_out, ["sub_layers"] = layerRepo, + ["connections"] = connections_t}) + + self:check_dim_len(3, 2) -- x, h, c and h, c +end + +function LSTMLayerT:init(batch_size, chunk_size) + self.dagL:init(batch_size, chunk_size) +end + +function LSTMLayerT:batch_resize(batch_size, chunk_size) + self.dagL:batch_resize(batch_size, chunk_size) +end + +function LSTMLayerT:update(bp_err, input, output, t) + self.dagL:update(bp_err, input, output, t) +end + +function LSTMLayerT:propagate(input, output, t) + self.dagL:propagate(input, output, t) +end + +function LSTMLayerT:back_propagate(bp_err, next_bp_err, input, output, t) + self.dagL:back_propagate(bp_err, next_bp_err, input, output, t) +end + +function LSTMLayerT:get_params() + return self.dagL:get_params() +end diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index 08e4849..d5408ba 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -343,11 +343,11 @@ end nerv.LMUtil.wait(2) nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) -nerv.printf("lr_half:%s\n", tostring(lr_half)) -nerv.printf("start_iter:%s\n", tostring(start_iter)) -nerv.printf("ppl_last:%s\n", tostring(ppl_last)) -nerv.printf("commds_str:%s\n", commands_str) -nerv.printf("test_iter:%s\n", tostring(test_iter)) +nerv.printf("lr_half:\t%s\n", tostring(lr_half)) +nerv.printf("start_iter:\t%s\n", tostring(start_iter)) +nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) +nerv.printf("commds_str:\t%s\n", commands_str) +nerv.printf("test_iter:\t%s\n", tostring(test_iter)) nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) nerv.LMUtil.wait(2) ------------------printing options end------------------------------ diff --git a/nerv/examples/lmptb/lstmlm_v2_ptb_main.lua b/nerv/examples/lmptb/lstmlm_v2_ptb_main.lua new file mode 100644 index 0000000..a3d7584 --- /dev/null +++ b/nerv/examples/lmptb/lstmlm_v2_ptb_main.lua @@ -0,0 +1,470 @@ +require 'lmptb.lmvocab' +require 'lmptb.lmfeeder' +require 'lmptb.lmutil' +require 'lmptb.layer.init' +--require 'tnn.init' +require 'lmptb.lmseqreader' +require 'lm_trainer' +require 'lmptb.lstm_t_v2' + +--[[global function rename]]-- +--local printf = nerv.printf +local LMTrainer = nerv.LMTrainer +--[[global function rename ends]]-- + +--global_conf: table +--first_time: bool +--Returns: a ParamRepo +function prepare_parameters(global_conf, iter) + nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) + + global_conf.paramRepo = nerv.ParamRepo() + local paramRepo = global_conf.paramRepo + + if iter == -1 then --first time + nerv.printf("%s first time, prepare some pre-set parameters, and leaving other parameters to auto-generation...\n", global_conf.sche_log_pre) + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:close() + --[[ + ltp_ih = nerv.LinearTransParam("ltp_ih", global_conf) + ltp_ih.trans = global_conf.cumat_type(global_conf.vocab:size(), global_conf.hidden_size) --index 0 is for zero, others correspond to vocab index(starting from 1) + ltp_ih.trans:generate(global_conf.param_random) + + ltp_hh = nerv.LinearTransParam("ltp_hh", global_conf) + ltp_hh.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.hidden_size) + ltp_hh.trans:generate(global_conf.param_random) + + --ltp_ho = nerv.LinearTransParam("ltp_ho", global_conf) + --ltp_ho.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.vocab:size()) + --ltp_ho.trans:generate(global_conf.param_random) + + bp_h = nerv.BiasParam("bp_h", global_conf) + bp_h.trans = global_conf.cumat_type(1, global_conf.hidden_size) + bp_h.trans:generate(global_conf.param_random) + + --bp_o = nerv.BiasParam("bp_o", global_conf) + --bp_o.trans = global_conf.cumat_type(1, global_conf.vocab:size()) + --bp_o.trans:generate(global_conf.param_random) + + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:write_chunk(ltp_ih) + f:write_chunk(ltp_hh) + --f:write_chunk(ltp_ho) + f:write_chunk(bp_h) + --f:write_chunk(bp_o) + f:close() + ]]-- + return nil + end + + nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, global_conf.param_fn .. '.' .. tostring(iter)) + paramRepo:import({global_conf.param_fn .. '.' .. tostring(iter)}, nil, global_conf) + + nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) + + return nil +end + +--global_conf: table +--Returns: nerv.LayerRepo +function prepare_layers(global_conf) + nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) + + local pr = global_conf.paramRepo + + local du = false + + --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}} + --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} + + local layers = { + --["nerv.AffineRecurrentLayer"] = { + -- ["recurrentL1"] = recurrentLconfig, + --}, + + ["nerv.LSTMLayerTv2"] = { + ["lstmL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}, + }, + + ["nerv.DropoutLayerT"] = { + ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SelectLinearLayer"] = { + ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, + }, + + --["nerv.SigmoidLayer"] = { + -- ["sigmoidL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + --}, + + ["nerv.CombinerLayer"] = { + ["combinerL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + }, + + ["nerv.AffineLayer"] = { + ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, + }, + + ["nerv.SoftmaxCELayerT"] = { + ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, + }, + } + + for l = 2, global_conf.layer_num do + layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.LSTMLayerTv2"]["lstmL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}} + layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} + end + --[[ --we do not need those in the new tnn framework + printf("%s adding %d bptt layers...\n", global_conf.sche_log_pre, global_conf.bptt) + for i = 1, global_conf.bptt do + layers["nerv.IndRecurrentLayer"]["recurrentL" .. (i + 1)] = recurrentLconfig + layers["nerv.SigmoidLayer"]["sigmoidL" .. (i + 1)] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.SelectLinearLayer"]["selectL" .. (i + 1)] = {{["ltp"] = "ltp_ih"}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}}} + end + --]] + + local layerRepo = nerv.LayerRepo(layers, pr, global_conf) + nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) + return layerRepo +end + +--global_conf: table +--layerRepo: nerv.LayerRepo +--Returns: a nerv.TNN +function prepare_tnn(global_conf, layerRepo) + nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) + + --input: input_w, input_w, ... input_w_now, last_activation + local connections_t = { + {"[1]", "selectL1[1]", 0}, + + --{"selectL1[1]", "recurrentL1[1]", 0}, + --{"recurrentL1[1]", "sigmoidL1[1]", 0}, + --{"sigmoidL1[1]", "combinerL1[1]", 0}, + --{"combinerL1[1]", "recurrentL1[2]", 1}, + + {"selectL1[1]", "lstmL1[1]", 0}, + {"lstmL1[2]", "lstmL1[3]", 1}, + {"lstmL1[1]", "combinerL1[1]", 0}, + {"combinerL1[1]", "lstmL1[2]", 1}, + {"combinerL1[2]", "dropoutL1[1]", 0}, + + {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, + {"outputL[1]", "softmaxL[1]", 0}, + {"[2]", "softmaxL[2]", 0}, + {"softmaxL[1]", "[1]", 0} + } + + for l = 2, global_conf.layer_num do + table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "lstmL"..l.."[1]", 0}) + table.insert(connections_t, {"lstmL"..l.."[2]", "lstmL"..l.."[3]", 1}) + table.insert(connections_t, {"lstmL"..l.."[1]", "combinerL"..l.."[1]", 0}) + table.insert(connections_t, {"combinerL"..l.."[1]", "lstmL"..l.."[2]", 1}) + table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) + end + + --[[ + printf("%s printing DAG connections:\n", global_conf.sche_log_pre) + for key, value in pairs(connections_t) do + printf("\t%s->%s\n", key, value) + end + ]]-- + + local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, + ["dim_out"] = {1}, ["sub_layers"] = layerRepo, + ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, + }) + + tnn:init(global_conf.batch_size, global_conf.chunk_size) + + nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) + return tnn +end + +function load_net(global_conf, next_iter) + prepare_parameters(global_conf, next_iter) + local layerRepo = prepare_layers(global_conf) + local tnn = prepare_tnn(global_conf, layerRepo) + return tnn +end + +local train_fn, valid_fn, test_fn +global_conf = {} +local set = arg[1] --"test" + +if (set == "ptb") then + +root_dir = '/home/slhome/txh18/workspace' +data_dir = root_dir .. '/ptb/DATA' +train_fn = data_dir .. '/ptb.train.txt.adds' +valid_fn = data_dir .. '/ptb.valid.txt.adds' +test_fn = data_dir .. '/ptb.test.txt.adds' +vocab_fn = data_dir .. '/vocab' + +qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 20, + max_iter = 45, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0.5", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/ptb/EXP-nerv/lstmlm_v2.0' +} + +elseif (set == "msr_sc") then + +data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' +train_fn = data_dir .. '/normed_all.sf.len60.adds.train' +valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' +test_fn = data_dir .. '/answer_normed.adds' +vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' + +global_conf = { + lrate = 1, wcost = 1e-6, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 10, + max_iter = 30, + decay_iter = 10, + lr_decay = 1.003, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 400000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' +} + +else + +valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' + +global_conf = { + lrate = 0.01, wcost = 1e-5, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 20, + layer_num = 1, + chunk_size = 2, + batch_size = 10, + max_iter = 3, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + lr_decay = 1.003, + decay_iter = 10, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 10, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_lstmlm_test' +} + +end + +local lr_half = false --can not be local, to be set by loadstring +local start_iter = -1 +local ppl_last = 100000 +local commands_str = "train:test" +local commands = {} +local test_iter = -1 + +--for testout(question) +local q_file = "ptb.test.txt.q10rs1_Msss.adds" + +if arg[2] ~= nil then + nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) + loadstring(arg[2])() + nerv.LMUtil.wait(0.5) +else + nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) +end + +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str +global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' +global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' +global_conf.param_fn = global_conf.work_dir .. "/params" +global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%X_%m_%d",os.time()) +commands = nerv.SUtil.parse_commands_set(commands_str) + +nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) +nerv.LMUtil.wait(2) +os.execute("mkdir -p "..global_conf.work_dir) +os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) + +--redirecting log outputs! +nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) + +----------------printing options--------------------------------- +nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) +for id, value in pairs(global_conf) do + nerv.printf("%s:\t%s\n", id, tostring(value)) +end +nerv.LMUtil.wait(2) + +nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) +nerv.printf("lr_half:\t%s\n", tostring(lr_half)) +nerv.printf("start_iter:\t%s\n", tostring(start_iter)) +nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) +nerv.printf("commds_str:\t%s\n", commands_str) +nerv.printf("test_iter:\t%s\n", tostring(test_iter)) +nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) +nerv.LMUtil.wait(2) +------------------printing options end------------------------------ + +math.randomseed(1) + +local vocab = nerv.LMVocab() +global_conf["vocab"] = vocab +nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) +global_conf.vocab:build_file(global_conf.vocab_fn, false) +ppl_rec = {} + +local final_iter = -1 +if commands["train"] == 1 then + if start_iter == -1 then + prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file + end + + if start_iter == -1 or start_iter == 0 then + nerv.printf("===INITIAL VALIDATION===\n") + local tnn = load_net(global_conf, 0) + global_conf.paramRepo = tnn:get_params() --get auto-generted params + global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file + global_conf.dropout_rate = 0 + local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + nerv.LMUtil.wait(1) + ppl_rec[0] = {} + ppl_rec[0].valid = result:ppl_all("rnn") + ppl_last = ppl_rec[0].valid + ppl_rec[0].train = 0 + ppl_rec[0].test = 0 + ppl_rec[0].lr = 0 + + start_iter = 1 + + nerv.printf("\n") + end + + for iter = start_iter, global_conf.max_iter, 1 do + final_iter = iter --for final testing + global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" + tnn = load_net(global_conf, iter - 1) + nerv.printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) + global_conf.dropout_rate = nerv.SUtil.sche_get(global_conf.dropout_list, iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! + global_conf.dropout_rate = 0 + ppl_rec[iter] = {} + ppl_rec[iter].train = result:ppl_all("rnn") + --shuffling training file + nerv.printf("%s shuffling training file\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) + os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) + nerv.printf("===PEEK ON TEST %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! + ppl_rec[iter].test = result:ppl_all("rnn") + nerv.printf("===VALIDATION %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + ppl_rec[iter].valid = result:ppl_all("rnn") + ppl_rec[iter].lr = global_conf.lrate + if ((ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true) and iter > global_conf.decay_iter) then + global_conf.lrate = (global_conf.lrate * 0.6) + end + if ppl_rec[iter].valid < ppl_last then + nerv.printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) + global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) + else + nerv.printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) + end + if ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true then + lr_half = true + end + if ppl_rec[iter].valid < ppl_last then + ppl_last = ppl_rec[iter].valid + end + nerv.printf("\n") + nerv.LMUtil.wait(2) + end + nerv.info("saving final nn to param.final") + os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final') + + nerv.printf("===VALIDATION PPL record===\n") + for i, _ in pairs(ppl_rec) do + nerv.printf(" \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) + end + nerv.printf("\n") +end --if commands["train"] + +if commands["test"] == 1 then + nerv.printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["test"] + +if commands["testout"] == 1 then + nerv.printf("===TEST OUT===\n") + nerv.printf("q_file:\t%s\n", q_file) + local q_fn = qdata_dir .. q_file + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false) --false update! +end --if commands["testout"] + + diff --git a/nerv/layer/gate_fff.lua b/nerv/layer/gate_fff.lua index 751dde1..6082e27 100644 --- a/nerv/layer/gate_fff.lua +++ b/nerv/layer/gate_fff.lua @@ -1,36 +1,33 @@ -local GateFFFLayer = nerv.class('nerv.GateFFFLayer', 'nerv.Layer') +local GateFFFLayer = nerv.class('nerv.GateFLayer', 'nerv.Layer') --Full matrix gate function GateFFFLayer:__init(id, global_conf, layer_conf) self.id = id self.dim_in = layer_conf.dim_in self.dim_out = layer_conf.dim_out self.gconf = global_conf - - self.ltp1 = self:find_param("ltp1", layer_conf, global_conf, nerv.LinearTransParam, {self.dim_in[1], self.dim_out[1]}) --layer_conf.ltp - self.ltp2 = self:find_param("ltp2", layer_conf, global_conf, nerv.LinearTransParam, {self.dim_in[2], self.dim_out[1]}) --layer_conf.ltp - self.ltp3 = self:find_param("ltp3", layer_conf, global_conf, nerv.LinearTransParam, {self.dim_in[3], self.dim_out[1]}) --layer_conf.ltp + + for i = 1, #self.dim_in do + self["ltp" .. i] = self:find_param("ltp" .. i, layer_conf, global_conf, nerv.LinearTransParam, {self.dim_in[i], self.dim_out[1]}) --layer_conf.ltp + end self.bp = self:find_param("bp", layer_conf, global_conf, nerv.BiasParam, {1, self.dim_out[1]})--layer_conf.bp - self:check_dim_len(3, 1) -- exactly one input and one output + self:check_dim_len(-1, 1) --accept multiple inputs end function GateFFFLayer:init(batch_size) - if self.ltp1.trans:ncol() ~= self.bp.trans:ncol() or - self.ltp2.trans:ncol() ~= self.bp.trans:ncol() or - self.ltp3.trans:ncol() ~= self.bp.trans:ncol() then - nerv.error("mismatching dimensions of linear transform and bias paramter") - end - if self.dim_in[1] ~= self.ltp1.trans:nrow() or - self.dim_in[2] ~= self.ltp2.trans:nrow() or - self.dim_in[3] ~= self.ltp3.trans:nrow() then - nerv.error("mismatching dimensions of linear transform parameter and input") + for i = 1, #self.dim_in do + if self["ltp" .. i].trans:ncol() ~= self.bp.trans:ncol() then + nerv.error("mismatching dimensions of linear transform and bias paramter") + end + if self.dim_in[i] ~= self["ltp" .. i].trans:nrow() then + nerv.error("mismatching dimensions of linear transform parameter and input") + end + self["ltp"..i]:train_init() end + if self.dim_out[1] ~= self.ltp1.trans:ncol() then nerv.error("mismatching dimensions of linear transform parameter and output") end - self.ltp1:train_init() - self.ltp2:train_init() - self.ltp3:train_init() self.bp:train_init() self.err_bakm = self.gconf.cumat_type(batch_size, self.dim_out[1]) end @@ -44,8 +41,9 @@ end function GateFFFLayer:propagate(input, output) -- apply linear transform output[1]:mul(input[1], self.ltp1.trans, 1.0, 0.0, 'N', 'N') - output[1]:mul(input[2], self.ltp2.trans, 1.0, 1.0, 'N', 'N') - output[1]:mul(input[3], self.ltp3.trans, 1.0, 1.0, 'N', 'N') + for i = 2, #self.dim_in do + output[1]:mul(input[i], self["ltp" .. i].trans, 1.0, 1.0, 'N', 'N') + end -- add bias output[1]:add_row(self.bp.trans, 1.0) output[1]:sigmoid(output[1]) @@ -53,19 +51,23 @@ end function GateFFFLayer:back_propagate(bp_err, next_bp_err, input, output) self.err_bakm:sigmoid_grad(bp_err[1], output[1]) - next_bp_err[1]:mul(self.err_bakm, self.ltp1.trans, 1.0, 0.0, 'N', 'T') - next_bp_err[2]:mul(self.err_bakm, self.ltp2.trans, 1.0, 0.0, 'N', 'T') - next_bp_err[3]:mul(self.err_bakm, self.ltp3.trans, 1.0, 0.0, 'N', 'T') + for i = 1, #self.dim_in do + next_bp_err[i]:mul(self.err_bakm, self["ltp" .. i].trans, 1.0, 0.0, 'N', 'T') + end end function GateFFFLayer:update(bp_err, input, output) self.err_bakm:sigmoid_grad(bp_err[1], output[1]) - self.ltp1:update_by_err_input(self.err_bakm, input[1]) - self.ltp2:update_by_err_input(self.err_bakm, input[2]) - self.ltp3:update_by_err_input(self.err_bakm, input[3]) + for i = 1, #self.dim_in do + self["ltp" .. i]:update_by_err_input(self.err_bakm, input[i]) + end self.bp:update_by_gradient(self.err_bakm:colsum()) end function GateFFFLayer:get_params() - return nerv.ParamRepo({self.ltp1, self.ltp2, self.ltp3, self.bp}) + local pr = nerv.ParamRepo({self.bp}) + for i = 1, #self.dim_in do + pr:add(self["ltp" .. i].id, self["ltp" .. i]) + end + return pr end diff --git a/nerv/tnn/layersT/lstm_t.lua b/nerv/tnn/layersT/lstm_t.lua index ded6058..2a3342d 100644 --- a/nerv/tnn/layersT/lstm_t.lua +++ b/nerv/tnn/layersT/lstm_t.lua @@ -36,7 +36,7 @@ function LSTMLayerT:__init(id, global_conf, layer_conf) [ap("mainTanhL")] = {{}, {["dim_in"] = {self.dim_out[1]}, ["dim_out"] = {self.dim_out[1]}}}, [ap("outputTanhL")] = {{}, {["dim_in"] = {self.dim_out[1]}, ["dim_out"] = {self.dim_out[1]}}}, }, - ["nerv.GateFFFLayer"] = { + ["nerv.GateFLayer"] = { [ap("forgetGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}}, [ap("inputGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2], self.dim_in[3]}, -- cgit v1.2.3-70-g09d2 From cba54520413b4abb92d5c64d0b5c135dd9b9f1ef Mon Sep 17 00:00:00 2001 From: txh18 Date: Fri, 4 Dec 2015 21:55:20 +0800 Subject: small bug fix for lmptb.lstm_t_v2 --- nerv/examples/lmptb/lmptb/lstm_t_v2.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerv/examples/lmptb/lmptb/lstm_t_v2.lua b/nerv/examples/lmptb/lmptb/lstm_t_v2.lua index d3fc920..dc2fe45 100644 --- a/nerv/examples/lmptb/lmptb/lstm_t_v2.lua +++ b/nerv/examples/lmptb/lmptb/lstm_t_v2.lua @@ -30,7 +30,7 @@ function LSTMLayerT:__init(id, global_conf, layer_conf) ["dim_out"] = {self.dim_in[3], self.dim_in[3]}, ["lambda"] = {1, 1}}}, }, ["nerv.AffineLayer"] = { - [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2], self.dim_in[3]}, + [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, ["dim_out"] = {self.dim_out[1]}, ["pr"] = pr}}, }, ["nerv.TanhLayer"] = { -- cgit v1.2.3-70-g09d2 From 2daed79a165015f164a46117dd7d8aa9cbfe5587 Mon Sep 17 00:00:00 2001 From: txh18 Date: Sat, 5 Dec 2015 13:37:01 +0800 Subject: changed lstm_t to a more standard version --- nerv/examples/lmptb/lstmlm_ptb_main.lua | 13 +++---------- nerv/tnn/layersT/lstm_t.lua | 11 +++++------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index d5408ba..a2dcbba 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -77,10 +77,6 @@ function prepare_layers(global_conf) --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} local layers = { - --["nerv.AffineRecurrentLayer"] = { - -- ["recurrentL1"] = recurrentLconfig, - --}, - ["nerv.LSTMLayerT"] = { ["lstmL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}, }, @@ -93,10 +89,6 @@ function prepare_layers(global_conf) ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, }, - --["nerv.SigmoidLayer"] = { - -- ["sigmoidL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} - --}, - ["nerv.CombinerLayer"] = { ["combinerL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, }, @@ -214,7 +206,7 @@ global_conf = { layer_num = 1, chunk_size = 15, batch_size = 20, - max_iter = 45, + max_iter = 35, lr_decay = 1.003, decay_iter = 10, param_random = function() return (math.random() / 5 - 0.1) end, @@ -323,7 +315,8 @@ global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' global_conf.param_fn = global_conf.work_dir .. "/params" global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) -global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%X_%m_%d",os.time()) +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) +global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') commands = nerv.SUtil.parse_commands_set(commands_str) nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) diff --git a/nerv/tnn/layersT/lstm_t.lua b/nerv/tnn/layersT/lstm_t.lua index 2a3342d..04d0600 100644 --- a/nerv/tnn/layersT/lstm_t.lua +++ b/nerv/tnn/layersT/lstm_t.lua @@ -24,12 +24,12 @@ function LSTMLayerT:__init(id, global_conf, layer_conf) [ap("inputHDup")] = {{}, {["dim_in"] = {self.dim_in[2]}, ["dim_out"] = {self.dim_in[2], self.dim_in[2], self.dim_in[2], self.dim_in[2]}, ["lambda"] = {1}}}, [ap("inputCDup")] = {{}, {["dim_in"] = {self.dim_in[3]}, - ["dim_out"] = {self.dim_in[3], self.dim_in[3], self.dim_in[3], self.dim_in[3]}, ["lambda"] = {1}}}, + ["dim_out"] = {self.dim_in[3], self.dim_in[3], self.dim_in[3]}, ["lambda"] = {1}}}, [ap("mainCDup")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3], self.dim_in[3], self.dim_in[3]}, ["lambda"] = {1, 1}}}, }, ["nerv.AffineLayer"] = { - [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2], self.dim_in[3]}, + [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, ["dim_out"] = {self.dim_out[1]}, ["pr"] = pr}}, }, ["nerv.TanhLayer"] = { @@ -61,21 +61,20 @@ function LSTMLayerT:__init(id, global_conf, layer_conf) [ap("inputXDup[1]")] = ap("mainAffineL[1]"), [ap("inputHDup[1]")] = ap("mainAffineL[2]"), - [ap("inputCDup[1]")] = ap("mainAffineL[3]"), [ap("mainAffineL[1]")] = ap("mainTanhL[1]"), [ap("inputXDup[2]")] = ap("inputGateL[1]"), [ap("inputHDup[2]")] = ap("inputGateL[2]"), - [ap("inputCDup[2]")] = ap("inputGateL[3]"), + [ap("inputCDup[1]")] = ap("inputGateL[3]"), [ap("inputXDup[3]")] = ap("forgetGateL[1]"), [ap("inputHDup[3]")] = ap("forgetGateL[2]"), - [ap("inputCDup[3]")] = ap("forgetGateL[3]"), + [ap("inputCDup[2]")] = ap("forgetGateL[3]"), [ap("mainTanhL[1]")] = ap("inputGMulL[1]"), [ap("inputGateL[1]")] = ap("inputGMulL[2]"), - [ap("inputCDup[4]")] = ap("forgetGMulL[1]"), + [ap("inputCDup[3]")] = ap("forgetGMulL[1]"), [ap("forgetGateL[1]")] = ap("forgetGMulL[2]"), [ap("inputGMulL[1]")] = ap("mainCDup[1]"), -- cgit v1.2.3-70-g09d2 From 902e547326311bab9d6494cebbc1e5e2f14a018b Mon Sep 17 00:00:00 2001 From: txh18 Date: Sat, 5 Dec 2015 17:47:30 +0800 Subject: added twitter, added bilstmlm script, todo: test bilstmlm --- nerv/examples/lmptb/bilstmlm_ptb_main.lua | 516 ++++++++++++++++++++++++++++++ nerv/examples/lmptb/lm_trainer.lua | 132 +++++++- nerv/examples/lmptb/lstmlm_ptb_main.lua | 47 ++- nerv/examples/lmptb/m-tests/some-text-chn | 5 + 4 files changed, 695 insertions(+), 5 deletions(-) create mode 100644 nerv/examples/lmptb/bilstmlm_ptb_main.lua create mode 100644 nerv/examples/lmptb/m-tests/some-text-chn diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua new file mode 100644 index 0000000..48bc636 --- /dev/null +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -0,0 +1,516 @@ +require 'lmptb.lmvocab' +require 'lmptb.lmfeeder' +require 'lmptb.lmutil' +require 'lmptb.layer.init' +--require 'tnn.init' +require 'lmptb.lmseqreader' +require 'lm_trainer' + +--[[global function rename]]-- +--local printf = nerv.printf +local LMTrainer = nerv.LMTrainer +--[[global function rename ends]]-- + +--global_conf: table +--first_time: bool +--Returns: a ParamRepo +function prepare_parameters(global_conf, iter) + nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) + + global_conf.paramRepo = nerv.ParamRepo() + local paramRepo = global_conf.paramRepo + + if iter == -1 then --first time + nerv.printf("%s first time, prepare some pre-set parameters, and leaving other parameters to auto-generation...\n", global_conf.sche_log_pre) + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:close() + --[[ + ltp_ih = nerv.LinearTransParam("ltp_ih", global_conf) + ltp_ih.trans = global_conf.cumat_type(global_conf.vocab:size(), global_conf.hidden_size) --index 0 is for zero, others correspond to vocab index(starting from 1) + ltp_ih.trans:generate(global_conf.param_random) + + ltp_hh = nerv.LinearTransParam("ltp_hh", global_conf) + ltp_hh.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.hidden_size) + ltp_hh.trans:generate(global_conf.param_random) + + --ltp_ho = nerv.LinearTransParam("ltp_ho", global_conf) + --ltp_ho.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.vocab:size()) + --ltp_ho.trans:generate(global_conf.param_random) + + bp_h = nerv.BiasParam("bp_h", global_conf) + bp_h.trans = global_conf.cumat_type(1, global_conf.hidden_size) + bp_h.trans:generate(global_conf.param_random) + + --bp_o = nerv.BiasParam("bp_o", global_conf) + --bp_o.trans = global_conf.cumat_type(1, global_conf.vocab:size()) + --bp_o.trans:generate(global_conf.param_random) + + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:write_chunk(ltp_ih) + f:write_chunk(ltp_hh) + --f:write_chunk(ltp_ho) + f:write_chunk(bp_h) + --f:write_chunk(bp_o) + f:close() + ]]-- + return nil + end + + nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, global_conf.param_fn .. '.' .. tostring(iter)) + paramRepo:import({global_conf.param_fn .. '.' .. tostring(iter)}, nil, global_conf) + + nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) + + return nil +end + +--global_conf: table +--Returns: nerv.LayerRepo +function prepare_layers(global_conf) + nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) + + local pr = global_conf.paramRepo + + local du = false + + --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}} + --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} + + local layers = { + ["nerv.LSTMLayerT"] = { + ["lstmFL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}, + ["lstmRL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}, + }, + + ["nerv.DropoutLayerT"] = { + ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SelectLinearLayer"] = { + ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, + }, + + ["nerv.CombinerLayer"] = { + ["combinerXL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + ["combinerHFL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + ["combinerHRL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + }, + + ["nerv.AffineLayer"] = { + ["biAffineL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["lambda"] = {1, 1}}}, + ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, + }, + + ["nerv.TanhLayer"] = { + ["biTanhL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SoftmaxCELayerT"] = { + ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, + }, + } + + if global_conf.layer_num > 1 then + nerv.error("this script currently do not support more than one layer") + end + --[[ + for l = 2, global_conf.layer_num do + layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.LSTMLayerT"]["lstmL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}} + layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} + end + ]]-- + + local layerRepo = nerv.LayerRepo(layers, pr, global_conf) + nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) + return layerRepo +end + +--global_conf: table +--layerRepo: nerv.LayerRepo +--Returns: a nerv.TNN +function prepare_tnn(global_conf, layerRepo) + nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) + + --input: input_w, input_w, ... input_w_now, last_activation + local connections_t = { + {"[1]", "selectL1[1]", 0}, + + --{"selectL1[1]", "recurrentL1[1]", 0}, + --{"recurrentL1[1]", "sigmoidL1[1]", 0}, + --{"sigmoidL1[1]", "combinerL1[1]", 0}, + --{"combinerL1[1]", "recurrentL1[2]", 1}, + + {"selectL1[1]", "combinerXL1[1]", 0}, + {"combinerXL1[1]", "lstmFL1[1]", 0}, + {"lstmFL1[1]", "combinerHFL1[1]", 0}, + {"combinerHFL1[1]", "lstmFL1[2]", 1}, + {"lstmFL1[2]", "lstmFL1[3]", 1}, + {"combinerXL1[2]", "lstmRL1[1]", 0}, + {"lstmRL1[1]", "combinerHRL1[1]", 0}, + {"combinerHRL1[1]", "lstmRL1[2]", -1}, + {"lstmRL1[2]", "lstmRL1[3]", -1}, + {"combinerHFL1[2]", "biAffineL1[1]", 0}, + {"combinerHRL1[2]", "biAffineL1[2]", 0}, + {"biAffineL1[1]", "biTanhL1[1]", 0}, + {"biTanhL1[1]", "dropoutL1[1]", 0}, + + {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, + {"outputL[1]", "softmaxL[1]", 0}, + {"[2]", "softmaxL[2]", 0}, + {"softmaxL[1]", "[1]", 0} + } + + --[[ + for l = 2, global_conf.layer_num do + table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "lstmL"..l.."[1]", 0}) + table.insert(connections_t, {"lstmL"..l.."[2]", "lstmL"..l.."[3]", 1}) + table.insert(connections_t, {"lstmL"..l.."[1]", "combinerL"..l.."[1]", 0}) + table.insert(connections_t, {"combinerL"..l.."[1]", "lstmL"..l.."[2]", 1}) + table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) + end + ]]-- + + --[[ + printf("%s printing DAG connections:\n", global_conf.sche_log_pre) + for key, value in pairs(connections_t) do + printf("\t%s->%s\n", key, value) + end + ]]-- + + local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, + ["dim_out"] = {1}, ["sub_layers"] = layerRepo, + ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, + }) + + tnn:init(global_conf.batch_size, global_conf.chunk_size) + + nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) + return tnn +end + +function load_net(global_conf, next_iter) + prepare_parameters(global_conf, next_iter) + local layerRepo = prepare_layers(global_conf) + local tnn = prepare_tnn(global_conf, layerRepo) + return tnn +end + +local train_fn, valid_fn, test_fn +global_conf = {} +local set = arg[1] --"test" + +if (set == "ptb") then + +root_dir = '/home/slhome/txh18/workspace' +data_dir = root_dir .. '/ptb/DATA' +train_fn = data_dir .. '/ptb.train.txt.adds' +valid_fn = data_dir .. '/ptb.valid.txt.adds' +test_fn = data_dir .. '/ptb.test.txt.adds' +vocab_fn = data_dir .. '/vocab' + +qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 90, + batch_size = 20, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0.5", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/ptb/EXP-nerv/bilstmlm_v1.0' +} + +elseif (set == "msr_sc") then + +data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' +train_fn = data_dir .. '/normed_all.sf.len60.adds.train' +valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' +test_fn = data_dir .. '/answer_normed.adds' +vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' + +global_conf = { + lrate = 1, wcost = 1e-6, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 10, + max_iter = 30, + decay_iter = 10, + lr_decay = 1.003, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 400000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' +} + +elseif (set == "twitter") then + +root_dir = '/home/slhome/txh18/workspace' +data_dir = root_dir .. '/twitter_new/DATA' +train_fn = data_dir .. '/twitter.choose.adds' +valid_fn = data_dir .. '/twitter.valid.adds' +test_fn = data_dir .. '/comm.test.choose-ppl.adds' +vocab_fn = data_dir .. '/twitter.choose.train.vocab' + +--qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 20, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/twitter_new/EXP-nerv/bilstmlm_v1.0' +} + +else + +valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' + +global_conf = { + lrate = 0.01, wcost = 1e-5, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 20, + layer_num = 1, + chunk_size = 2, + batch_size = 10, + max_iter = 3, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + max_sen_len = 80, + lr_decay = 1.003, + decay_iter = 10, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 10, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_bilstmlm_test' +} + +end + +local lr_half = false --can not be local, to be set by loadstring +local start_iter = -1 +local ppl_last = 100000 +local commands_str = "train:test" +local commands = {} +local test_iter = -1 + +--for testout(question) +local q_file = "ptb.test.txt.q10rs1_Msss.adds" + +if arg[2] ~= nil then + nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) + loadstring(arg[2])() + nerv.LMUtil.wait(0.5) +else + nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) +end + +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str +global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' +global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' +global_conf.param_fn = global_conf.work_dir .. "/params" +global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) +global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') +commands = nerv.SUtil.parse_commands_set(commands_str) + +nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) +nerv.LMUtil.wait(2) +os.execute("mkdir -p "..global_conf.work_dir) +os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) + +--redirecting log outputs! +nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) + +----------------printing options--------------------------------- +nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) +for id, value in pairs(global_conf) do + nerv.printf("%s:\t%s\n", id, tostring(value)) +end +nerv.LMUtil.wait(2) + +nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) +nerv.printf("lr_half:\t%s\n", tostring(lr_half)) +nerv.printf("start_iter:\t%s\n", tostring(start_iter)) +nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) +nerv.printf("commds_str:\t%s\n", commands_str) +nerv.printf("test_iter:\t%s\n", tostring(test_iter)) +nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) +nerv.LMUtil.wait(2) +------------------printing options end------------------------------ + +math.randomseed(1) + +local vocab = nerv.LMVocab() +global_conf["vocab"] = vocab +nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) +global_conf.vocab:build_file(global_conf.vocab_fn, false) +ppl_rec = {} + +local final_iter = -1 +if commands["train"] == 1 then + if start_iter == -1 then + prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file + end + + if start_iter == -1 or start_iter == 0 then + nerv.printf("===INITIAL VALIDATION===\n") + local tnn = load_net(global_conf, 0) + global_conf.paramRepo = tnn:get_params() --get auto-generted params + global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file + global_conf.dropout_rate = 0 + local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + nerv.LMUtil.wait(1) + ppl_rec[0] = {} + ppl_rec[0].valid = result:ppl_all("rnn") + ppl_last = ppl_rec[0].valid + ppl_rec[0].train = 0 + ppl_rec[0].test = 0 + ppl_rec[0].lr = 0 + + start_iter = 1 + + nerv.printf("\n") + end + + for iter = start_iter, global_conf.max_iter, 1 do + final_iter = iter --for final testing + global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" + tnn = load_net(global_conf, iter - 1) + nerv.printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) + global_conf.dropout_rate = nerv.SUtil.sche_get(global_conf.dropout_list, iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! + global_conf.dropout_rate = 0 + ppl_rec[iter] = {} + ppl_rec[iter].train = result:ppl_all("rnn") + --shuffling training file + nerv.printf("%s shuffling training file\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) + os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) + nerv.printf("===PEEK ON TEST %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! + ppl_rec[iter].test = result:ppl_all("rnn") + nerv.printf("===VALIDATION %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + ppl_rec[iter].valid = result:ppl_all("rnn") + ppl_rec[iter].lr = global_conf.lrate + if ((ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true) and iter > global_conf.decay_iter) then + global_conf.lrate = (global_conf.lrate * 0.6) + end + if ppl_rec[iter].valid < ppl_last then + nerv.printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) + global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) + else + nerv.printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) + end + if ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true then + lr_half = true + end + if ppl_rec[iter].valid < ppl_last then + ppl_last = ppl_rec[iter].valid + end + nerv.printf("\n") + nerv.LMUtil.wait(2) + end + nerv.info("saving final nn to param.final") + os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final') + + nerv.printf("===VALIDATION PPL record===\n") + for i, _ in pairs(ppl_rec) do + nerv.printf(" \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) + end + nerv.printf("\n") +end --if commands["train"] + +if commands["test"] == 1 then + nerv.printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["test"] + +if commands["testout"] == 1 then + nerv.printf("===TEST OUT===\n") + nerv.printf("q_file:\t%s\n", q_file) + local q_fn = qdata_dir .. q_file + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false) --false update! +end --if commands["testout"] + + diff --git a/nerv/examples/lmptb/lm_trainer.lua b/nerv/examples/lmptb/lm_trainer.lua index 3c7078e..6bd06bb 100644 --- a/nerv/examples/lmptb/lm_trainer.lua +++ b/nerv/examples/lmptb/lm_trainer.lua @@ -22,6 +22,7 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) p_conf = {} end local reader + local r_conf local chunk_size, batch_size if p_conf.one_sen_report == true then --report log prob one by one sentence if do_train == true then @@ -31,12 +32,13 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) global_conf.max_sen_len) batch_size = 1 chunk_size = global_conf.max_sen_len + r_conf["se_mode"] = true else batch_size = global_conf.batch_size chunk_size = global_conf.chunk_size end - reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, global_conf.vocab) + reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, global_conf.vocab, r_conf) reader:open_file(fn) local result = nerv.LMResult(global_conf, global_conf.vocab) @@ -144,4 +146,132 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) return result end +--Returns: LMResult +function LMTrainer.lm_process_file_birnn(global_conf, fn, tnn, do_train, p_conf) + if p_conf == nil then + p_conf = {} + end + local reader + local chunk_size, batch_size + local r_conf = {["se_mode"] = true} + if p_conf.one_sen_report == true then --report log prob one by one sentence + if do_train == true then + nerv.warning("LMTrainer.lm_process_file_birnn: warning, one_sen_report is true while do_train is also true, strange") + end + nerv.printf("lm_process_file_birnn: one_sen report mode, set batch_size to 1 and chunk_size to max_sen_len(%d)\n", + global_conf.max_sen_len) + batch_size = 1 + chunk_size = global_conf.max_sen_len + else + batch_size = global_conf.batch_size + chunk_size = global_conf.chunk_size + end + + reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, global_conf.vocab) + reader:open_file(fn) + + local result = nerv.LMResult(global_conf, global_conf.vocab) + result:init("birnn") + if global_conf.dropout_rate ~= nil then + nerv.info("LMTrainer.lm_process_file_birnn: dropout_rate is %f", global_conf.dropout_rate) + end + + global_conf.timer:flush() + tnn:flush_all() --caution: will also flush the inputs from the reader! + + local next_log_wcn = global_conf.log_w_num + local neto_bakm = global_conf.mmat_type(batch_size, 1) --space backup matrix for network output + + while (1) do + global_conf.timer:tic('most_out_loop_lmprocessfile') + + local r, feeds + global_conf.timer:tic('tnn_beforeprocess') + r, feeds = tnn:getfeed_from_reader(reader) + if r == false then + break + end + + for t = 1, chunk_size do + tnn.err_inputs_m[t][1]:fill(1) + for i = 1, batch_size do + if bit.band(feeds.flags_now[t][i], nerv.TNN.FC.HAS_LABEL) == 0 then + tnn.err_inputs_m[t][1][i - 1][0] = 0 + end + end + end + global_conf.timer:toc('tnn_beforeprocess') + + --[[ + for j = 1, global_conf.chunk_size, 1 do + for i = 1, global_conf.batch_size, 1 do + printf("%s[L(%s)] ", feeds.inputs_s[j][i], feeds.labels_s[j][i]) --vocab:get_word_str(input[i][j]).id + end + printf("\n") + end + printf("\n") + ]]-- + + tnn:net_propagate() + + if do_train == true then + tnn:net_backpropagate(false) + tnn:net_backpropagate(true) + end + + global_conf.timer:tic('tnn_afterprocess') + local sen_logp = {} + for t = 1, chunk_size, 1 do + tnn.outputs_m[t][1]:copy_toh(neto_bakm) + for i = 1, batch_size, 1 do + if (feeds.labels_s[t][i] ~= global_conf.vocab.null_token) then + result:add("birnn", feeds.labels_s[t][i], math.exp(neto_bakm[i - 1][0])) + if sen_logp[i] == nil then + sen_logp[i] = 0 + end + sen_logp[i] = sen_logp[i] + neto_bakm[i - 1][0] + end + end + end + if p_conf.one_sen_report == true then + for i = 1, batch_size do + nerv.printf("LMTrainer.lm_process_file_birnn: one_sen_report, %f\n", sen_logp[i]) + end + end + + --tnn:move_right_to_nextmb({0}) --do not need history for bi directional model + global_conf.timer:toc('tnn_afterprocess') + + global_conf.timer:toc('most_out_loop_lmprocessfile') + + --print log + if result["birnn"].cn_w > next_log_wcn then + next_log_wcn = next_log_wcn + global_conf.log_w_num + nerv.printf("%s %d words processed %s.\n", global_conf.sche_log_pre, result["birnn"].cn_w, os.date()) + nerv.printf("\t%s log prob per sample :%f.\n", global_conf.sche_log_pre, result:logp_sample("birnn")) + for key, value in pairs(global_conf.timer.rec) do + nerv.printf("\t [global_conf.timer]: time spent on %s:%.5f clock time\n", key, value) + end + global_conf.timer:flush() + nerv.LMUtil.wait(0.1) + end + + --[[ + for t = 1, global_conf.chunk_size do + print(tnn.outputs_m[t][1]) + end + ]]-- + + collectgarbage("collect") + + --break --debug + end + + nerv.printf("%s Displaying result:\n", global_conf.sche_log_pre) + nerv.printf("%s %s\n", global_conf.sche_log_pre, result:status("birnn")) + nerv.printf("%s Doing on %s end.\n", global_conf.sche_log_pre, fn) + + return result +end + diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index a2dcbba..887993d 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -257,12 +257,50 @@ global_conf = { work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' } +elseif (set == "twitter") then + +root_dir = '/home/slhome/txh18/workspace' +data_dir = root_dir .. '/twitter_new/DATA' +train_fn = data_dir .. '/twitter.choose.adds' +valid_fn = data_dir .. '/twitter.valid.adds' +test_fn = data_dir .. '/comm.test.choose-ppl.adds' +vocab_fn = data_dir .. '/twitter.choose.train.vocab' + +--qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 20, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/twitter_new/EXP-nerv/lstmlm_v1.0' +} + else -valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' -train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' -test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' -vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' global_conf = { lrate = 0.01, wcost = 1e-5, momentum = 0, @@ -281,6 +319,7 @@ global_conf = { train_fn = train_fn, valid_fn = valid_fn, test_fn = test_fn, + max_sen_len = 80, lr_decay = 1.003, decay_iter = 10, vocab_fn = vocab_fn, diff --git a/nerv/examples/lmptb/m-tests/some-text-chn b/nerv/examples/lmptb/m-tests/some-text-chn new file mode 100644 index 0000000..da474ce --- /dev/null +++ b/nerv/examples/lmptb/m-tests/some-text-chn @@ -0,0 +1,5 @@ + 你好 我 是 一个 人 + 想 一起 玩 吗 + 一个 人 很 好 玩 + 不 想 一个 人 玩 + 不 想 一个 人 玩 -- cgit v1.2.3-70-g09d2 From ea0e37892ae70357305da3b1fbae617215a25778 Mon Sep 17 00:00:00 2001 From: txh18 Date: Sat, 5 Dec 2015 19:04:11 +0800 Subject: small bug fix in lm training script --- nerv/examples/lmptb/bilstmlm_ptb_main.lua | 14 +++++++------- nerv/examples/lmptb/lstmlm_ptb_main.lua | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua index 48bc636..9100d0d 100644 --- a/nerv/examples/lmptb/bilstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -346,12 +346,12 @@ global_conf = { end -local lr_half = false --can not be local, to be set by loadstring -local start_iter = -1 -local ppl_last = 100000 -local commands_str = "train:test" -local commands = {} -local test_iter = -1 +lr_half = false --can not be local, to be set by loadstring +start_iter = -1 +ppl_last = 100000 +commands_str = "train:test" +commands = {} +test_iter = -1 --for testout(question) local q_file = "ptb.test.txt.q10rs1_Msss.adds" @@ -393,7 +393,7 @@ nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log nerv.printf("lr_half:\t%s\n", tostring(lr_half)) nerv.printf("start_iter:\t%s\n", tostring(start_iter)) nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) -nerv.printf("commds_str:\t%s\n", commands_str) +nerv.printf("commands_str:\t%s\n", commands_str) nerv.printf("test_iter:\t%s\n", tostring(test_iter)) nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) nerv.LMUtil.wait(2) diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index 887993d..2751ec8 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -331,12 +331,12 @@ global_conf = { end -local lr_half = false --can not be local, to be set by loadstring -local start_iter = -1 -local ppl_last = 100000 -local commands_str = "train:test" -local commands = {} -local test_iter = -1 +lr_half = false --can not be local, to be set by loadstring +start_iter = -1 +ppl_last = 100000 +commands_str = "train:test" +commands = {} +test_iter = -1 --for testout(question) local q_file = "ptb.test.txt.q10rs1_Msss.adds" @@ -378,7 +378,7 @@ nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log nerv.printf("lr_half:\t%s\n", tostring(lr_half)) nerv.printf("start_iter:\t%s\n", tostring(start_iter)) nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) -nerv.printf("commds_str:\t%s\n", commands_str) +nerv.printf("commands_str:\t%s\n", commands_str) nerv.printf("test_iter:\t%s\n", tostring(test_iter)) nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) nerv.LMUtil.wait(2) -- cgit v1.2.3-70-g09d2 From 79c711d9c92a8e92f7ad9187a66d3e2aac239356 Mon Sep 17 00:00:00 2001 From: txh18 Date: Sun, 6 Dec 2015 13:33:26 +0800 Subject: small bug fix in lm training script --- nerv/examples/lmptb/lm_trainer.lua | 8 +++++--- nerv/examples/lmptb/lstmlm_ptb_main.lua | 10 +++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/nerv/examples/lmptb/lm_trainer.lua b/nerv/examples/lmptb/lm_trainer.lua index 6bd06bb..2cdbd4f 100644 --- a/nerv/examples/lmptb/lm_trainer.lua +++ b/nerv/examples/lmptb/lm_trainer.lua @@ -22,7 +22,7 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) p_conf = {} end local reader - local r_conf + local r_conf = {} local chunk_size, batch_size if p_conf.one_sen_report == true then --report log prob one by one sentence if do_train == true then @@ -48,6 +48,7 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) end global_conf.timer:flush() + tnn:init(batch_size, chunk_size) tnn:flush_all() --caution: will also flush the inputs from the reader! local next_log_wcn = global_conf.log_w_num @@ -107,7 +108,7 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) end if p_conf.one_sen_report == true then for i = 1, batch_size do - nerv.printf("LMTrainer.lm_process_file_rnn: one_sen_report, %f\n", sen_logp[i]) + nerv.printf("LMTrainer.lm_process_file_rnn: one_sen_report_output, %f\n", sen_logp[i]) end end @@ -177,6 +178,7 @@ function LMTrainer.lm_process_file_birnn(global_conf, fn, tnn, do_train, p_conf) end global_conf.timer:flush() + tnn:init(batch_size, chunk_size) tnn:flush_all() --caution: will also flush the inputs from the reader! local next_log_wcn = global_conf.log_w_num @@ -235,7 +237,7 @@ function LMTrainer.lm_process_file_birnn(global_conf, fn, tnn, do_train, p_conf) end if p_conf.one_sen_report == true then for i = 1, batch_size do - nerv.printf("LMTrainer.lm_process_file_birnn: one_sen_report, %f\n", sen_logp[i]) + nerv.printf("LMTrainer.lm_process_file_birnn: one_sen_report_output, %f\n", sen_logp[i]) end end diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index 2751ec8..804438d 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -337,9 +337,8 @@ ppl_last = 100000 commands_str = "train:test" commands = {} test_iter = -1 - --for testout(question) -local q_file = "ptb.test.txt.q10rs1_Msss.adds" +q_file = "ptb.test.txt.q10rs1_Msss.adds" if arg[2] ~= nil then nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) @@ -485,8 +484,8 @@ end --if commands["test"] if commands["testout"] == 1 then nerv.printf("===TEST OUT===\n") nerv.printf("q_file:\t%s\n", q_file) - local q_fn = qdata_dir .. q_file - global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + local q_fn = qdata_dir .. '/' .. q_file + global_conf.sche_log_pre = "[SCHEDULER TESTOUT]:" if final_iter ~= -1 and test_iter == -1 then test_iter = final_iter end @@ -495,7 +494,8 @@ if commands["testout"] == 1 then end tnn = load_net(global_conf, test_iter) global_conf.dropout_rate = 0 - LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false) --false update! + LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false, + {["one_sen_report"] = true}) --false update! end --if commands["testout"] -- cgit v1.2.3-70-g09d2 From 313011f24dfdacfe9c18d018d5bb877625a09ec7 Mon Sep 17 00:00:00 2001 From: txh18 Date: Sun, 6 Dec 2015 15:18:13 +0800 Subject: small bug fix on lm train script --- nerv/examples/lmptb/bilstmlm_ptb_main.lua | 23 +++++++++++------------ nerv/examples/lmptb/lm_trainer.lua | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua index 9100d0d..cf0009b 100644 --- a/nerv/examples/lmptb/bilstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -325,7 +325,7 @@ global_conf = { hidden_size = 20, layer_num = 1, - chunk_size = 2, + chunk_size = 20, batch_size = 10, max_iter = 3, param_random = function() return (math.random() / 5 - 0.1) end, @@ -372,7 +372,6 @@ global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') commands = nerv.SUtil.parse_commands_set(commands_str) - nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) nerv.LMUtil.wait(2) os.execute("mkdir -p "..global_conf.work_dir) @@ -419,10 +418,10 @@ if commands["train"] == 1 then global_conf.paramRepo = tnn:get_params() --get auto-generted params global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file global_conf.dropout_rate = 0 - local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + local result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.valid_fn, tnn, false) --false update! nerv.LMUtil.wait(1) ppl_rec[0] = {} - ppl_rec[0].valid = result:ppl_all("rnn") + ppl_rec[0].valid = result:ppl_all("birnn") ppl_last = ppl_rec[0].valid ppl_rec[0].train = 0 ppl_rec[0].test = 0 @@ -439,20 +438,20 @@ if commands["train"] == 1 then tnn = load_net(global_conf, iter - 1) nerv.printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) global_conf.dropout_rate = nerv.SUtil.sche_get(global_conf.dropout_list, iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! + result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! global_conf.dropout_rate = 0 ppl_rec[iter] = {} - ppl_rec[iter].train = result:ppl_all("rnn") + ppl_rec[iter].train = result:ppl_all("birnn") --shuffling training file nerv.printf("%s shuffling training file\n", global_conf.sche_log_pre) os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) nerv.printf("===PEEK ON TEST %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! - ppl_rec[iter].test = result:ppl_all("rnn") + result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.test_fn, tnn, false) --false update! + ppl_rec[iter].test = result:ppl_all("birnn") nerv.printf("===VALIDATION %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! - ppl_rec[iter].valid = result:ppl_all("rnn") + result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + ppl_rec[iter].valid = result:ppl_all("birnn") ppl_rec[iter].lr = global_conf.lrate if ((ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true) and iter > global_conf.decay_iter) then global_conf.lrate = (global_conf.lrate * 0.6) @@ -494,7 +493,7 @@ if commands["test"] == 1 then end tnn = load_net(global_conf, test_iter) global_conf.dropout_rate = 0 - LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! + LMTrainer.lm_process_file_birnn(global_conf, global_conf.test_fn, tnn, false) --false update! end --if commands["test"] if commands["testout"] == 1 then @@ -510,7 +509,7 @@ if commands["testout"] == 1 then end tnn = load_net(global_conf, test_iter) global_conf.dropout_rate = 0 - LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false) --false update! + LMTrainer.lm_process_file_birnn(global_conf, q_fn, tnn, false) --false update! end --if commands["testout"] diff --git a/nerv/examples/lmptb/lm_trainer.lua b/nerv/examples/lmptb/lm_trainer.lua index 2cdbd4f..0ccd847 100644 --- a/nerv/examples/lmptb/lm_trainer.lua +++ b/nerv/examples/lmptb/lm_trainer.lua @@ -168,7 +168,7 @@ function LMTrainer.lm_process_file_birnn(global_conf, fn, tnn, do_train, p_conf) chunk_size = global_conf.chunk_size end - reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, global_conf.vocab) + reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, global_conf.vocab, r_conf) reader:open_file(fn) local result = nerv.LMResult(global_conf, global_conf.vocab) -- cgit v1.2.3-70-g09d2 From 4e3cf34a29afbdb17c42ab7f1efacec52efd0e8b Mon Sep 17 00:00:00 2001 From: txh18 Date: Sun, 6 Dec 2015 19:18:07 +0800 Subject: added extend_t for tnn to save GPU memory --- nerv/examples/lmptb/bilstmlm_ptb_main.lua | 2 +- nerv/tnn/tnn.lua | 35 ++++++++++++++++++------------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua index cf0009b..2aba1d2 100644 --- a/nerv/examples/lmptb/bilstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -364,7 +364,7 @@ else nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) end -global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num --.. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' global_conf.param_fn = global_conf.work_dir .. "/params" diff --git a/nerv/tnn/tnn.lua b/nerv/tnn/tnn.lua index 56c9dc0..beb73ca 100644 --- a/nerv/tnn/tnn.lua +++ b/nerv/tnn/tnn.lua @@ -59,12 +59,12 @@ nerv.TNN.FC.HAS_INPUT = 1 nerv.TNN.FC.HAS_LABEL = 2 nerv.TNN.FC.SEQ_NORM = bit.bor(nerv.TNN.FC.HAS_INPUT, nerv.TNN.FC.HAS_LABEL) --This instance have both input and label -function TNN.make_initial_store(st, p, dim, batch_size, chunk_size, global_conf, st_c, p_c, t_c) - --Return a table of matrix storage from time (1-chunk_size)..(2*chunk_size) +function TNN.make_initial_store(st, p, dim, batch_size, chunk_size, extend_t, global_conf, st_c, p_c, t_c) + --Return a table of matrix storage from time (1-extend_t)..(chunk_size+extend_t) if (type(st) ~= "table") then nerv.error("st should be a table") end - for i = 1 - chunk_size - 1, chunk_size * 2 + 1 do --intentionally allocated more time, should be [1-chunk_size, chunk_size*2] + for i = 1 - extend_t - 1, chunk_size + extend_t + 1 do --intentionally allocated more time if (st[i] == nil) then st[i] = {} end @@ -97,6 +97,13 @@ function TNN:__init(id, global_conf, layer_conf) if self.clip_t > 0 then nerv.info("tnn(%s) will clip gradient across time with %f...", id, self.clip_t) end + + self.extend_t = layer_conf.extend_t --TNN will allocate storage of time for 1-extend_t .. chunk_size+extend_t + if self.extend_t == nil then + self.extend_t = 5 + end + nerv.info("tnn(%s) will extend storage beyond MB border for time steps %d...", id, self.extend_t) + local layers = {} local inputs_p = {} --map:port of the TNN to layer ref and port local outputs_p = {} @@ -173,9 +180,9 @@ function TNN:init(batch_size, chunk_size) print("TNN initing storage", ref_from.layer.id, "->", ref_to.layer.id) ref_to.inputs_matbak_p[port_to] = self.gconf.cumat_type(batch_size, dim) - self.make_initial_store(ref_from.outputs_m, port_from, dim, batch_size, chunk_size, self.gconf, ref_to.inputs_m, port_to, time) + self.make_initial_store(ref_from.outputs_m, port_from, dim, batch_size, chunk_size, self.extend_t, self.gconf, ref_to.inputs_m, port_to, time) ref_from.err_inputs_matbak_p[port_from] = self.gconf.cumat_type(batch_size, dim) - self.make_initial_store(ref_from.err_inputs_m, port_from, dim, batch_size, chunk_size, self.gconf, ref_to.err_outputs_m, port_to, time) + self.make_initial_store(ref_from.err_inputs_m, port_from, dim, batch_size, chunk_size, self.extend_t, self.gconf, ref_to.err_outputs_m, port_to, time) end @@ -184,8 +191,8 @@ function TNN:init(batch_size, chunk_size) for i = 1, #self.dim_out do --Init storage for output ports local ref = self.outputs_p[i].ref local p = self.outputs_p[i].port - self.make_initial_store(ref.outputs_m, p, self.dim_out[i], batch_size, chunk_size, self.gconf, self.outputs_m, i, 0) - self.make_initial_store(ref.err_inputs_m, p, self.dim_out[i], batch_size, chunk_size, self.gconf, self.err_inputs_m, i, 0) + self.make_initial_store(ref.outputs_m, p, self.dim_out[i], batch_size, chunk_size, self.extend_t, self.gconf, self.outputs_m, i, 0) + self.make_initial_store(ref.err_inputs_m, p, self.dim_out[i], batch_size, chunk_size, self.extend_t, self.gconf, self.err_inputs_m, i, 0) end self.inputs_m = {} @@ -193,8 +200,8 @@ function TNN:init(batch_size, chunk_size) for i = 1, #self.dim_in do --Init storage for input ports local ref = self.inputs_p[i].ref local p = self.inputs_p[i].port - self.make_initial_store(ref.inputs_m, p, self.dim_in[i], batch_size, chunk_size, self.gconf, self.inputs_m, i, 0) - self.make_initial_store(ref.err_outputs_m, p, self.dim_in[i], batch_size, chunk_size, self.gconf, self.err_outputs_m, i, 0) + self.make_initial_store(ref.inputs_m, p, self.dim_in[i], batch_size, chunk_size, self.extend_t, self.gconf, self.inputs_m, i, 0) + self.make_initial_store(ref.err_outputs_m, p, self.dim_in[i], batch_size, chunk_size, self.extend_t, self.gconf, self.err_outputs_m, i, 0) end for id, ref in pairs(self.layers) do --Calling init for child layers @@ -260,7 +267,7 @@ function TNN:flush_all() --flush all history and activation local _, ref for _, ref in pairs(self.layers) do for i = 1, #ref.dim_in do - for t = 1 - self.chunk_size, self.chunk_size * 2 do + for t = 1 - self.extend_t, self.chunk_size + self.extend_t do ref.inputs_m[t][i]:fill(self.gconf.nn_act_default) if (ref.inputs_b[t] == nil) then ref.inputs_b[t] = {} @@ -274,7 +281,7 @@ function TNN:flush_all() --flush all history and activation end end for i = 1, #ref.dim_out do - for t = 1 - self.chunk_size, self.chunk_size * 2 do + for t = 1 - self.extend_t, self.chunk_size + self.extend_t do ref.outputs_m[t][i]:fill(self.gconf.nn_act_default) if (ref.outputs_b[t] == nil) then ref.outputs_b[t] = {} @@ -302,13 +309,13 @@ end function TNN:move_right_to_nextmb(list_t) --move output history activations of 1..chunk_size to 1-chunk_size..0 if list_t == nil then list_t = {} - for i = 1, self.chunk_size do - list_t[i] = i - self.chunk_size + for i = self.extend_t, 1, -1 do + list_t[i] = 1 - i end end for i = 1, #list_t do t = list_t[i] - if t < 1 - self.chunk_size or t > 0 then + if t < 1 - self.extend_t or t > 0 then nerv.error("MB move range error") end for id, ref in pairs(self.layers) do -- cgit v1.2.3-70-g09d2 From 27cf4ef1965ec900bb15b32c2b8de65a65c0cd8b Mon Sep 17 00:00:00 2001 From: txh18 Date: Sun, 6 Dec 2015 19:31:38 +0800 Subject: some log chang --- nerv/examples/lmptb/lm_trainer.lua | 5 ++++- nerv/tnn/layer_dag_t.lua | 2 +- nerv/tnn/tnn.lua | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/nerv/examples/lmptb/lm_trainer.lua b/nerv/examples/lmptb/lm_trainer.lua index 0ccd847..3b8b5c3 100644 --- a/nerv/examples/lmptb/lm_trainer.lua +++ b/nerv/examples/lmptb/lm_trainer.lua @@ -53,7 +53,8 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) local next_log_wcn = global_conf.log_w_num local neto_bakm = global_conf.mmat_type(batch_size, 1) --space backup matrix for network output - + + nerv.info("LMTrainer.lm_process_file_rnn: begin processing...") while (1) do global_conf.timer:tic('most_out_loop_lmprocessfile') @@ -184,6 +185,8 @@ function LMTrainer.lm_process_file_birnn(global_conf, fn, tnn, do_train, p_conf) local next_log_wcn = global_conf.log_w_num local neto_bakm = global_conf.mmat_type(batch_size, 1) --space backup matrix for network output + nerv.info("LMTrainer.lm_process_file_birnn: begin processing...") + while (1) do global_conf.timer:tic('most_out_loop_lmprocessfile') diff --git a/nerv/tnn/layer_dag_t.lua b/nerv/tnn/layer_dag_t.lua index e3a9316..b651f4e 100644 --- a/nerv/tnn/layer_dag_t.lua +++ b/nerv/tnn/layer_dag_t.lua @@ -142,7 +142,7 @@ function DAGLayerT:__init(id, global_conf, layer_conf) end function DAGLayerT:init(batch_size, chunk_size) - nerv.info("initing DAGLayerT %s...\n", self.id) + nerv.info("initing DAGLayerT %s...", self.id) if chunk_size == nil then chunk_size = 1 nerv.info("(Initing DAGLayerT) chunk_size is nil, setting it to default 1\n") diff --git a/nerv/tnn/tnn.lua b/nerv/tnn/tnn.lua index beb73ca..cf02123 100644 --- a/nerv/tnn/tnn.lua +++ b/nerv/tnn/tnn.lua @@ -178,7 +178,7 @@ function TNN:init(batch_size, chunk_size) nerv.error("layer %s has a zero dim port", ref_from.layer.id) end - print("TNN initing storage", ref_from.layer.id, "->", ref_to.layer.id) + nerv.info("TNN initing storage %s->%s", ref_from.layer.id, ref_to.layer.id) ref_to.inputs_matbak_p[port_to] = self.gconf.cumat_type(batch_size, dim) self.make_initial_store(ref_from.outputs_m, port_from, dim, batch_size, chunk_size, self.extend_t, self.gconf, ref_to.inputs_m, port_to, time) ref_from.err_inputs_matbak_p[port_from] = self.gconf.cumat_type(batch_size, dim) -- cgit v1.2.3-70-g09d2 From 5e3f5996f053debe048e30e71d155e68bf6598b5 Mon Sep 17 00:00:00 2001 From: txh18 Date: Sun, 6 Dec 2015 21:36:43 +0800 Subject: lm script change:user can set start_lr --- nerv/examples/lmptb/bilstmlm_ptb_main.lua | 4 ++++ nerv/examples/lmptb/lstmlm_ptb_main.lua | 3 +++ 2 files changed, 7 insertions(+) diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua index 2aba1d2..078d036 100644 --- a/nerv/examples/lmptb/bilstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -348,6 +348,7 @@ end lr_half = false --can not be local, to be set by loadstring start_iter = -1 +start_lr = global_conf.lrate ppl_last = 100000 commands_str = "train:test" commands = {} @@ -372,6 +373,9 @@ global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') commands = nerv.SUtil.parse_commands_set(commands_str) + +global_conf.lrate = start_lr --starting lr can be set by user(arg[2]) + nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) nerv.LMUtil.wait(2) os.execute("mkdir -p "..global_conf.work_dir) diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index 804438d..b76fab8 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -333,6 +333,7 @@ end lr_half = false --can not be local, to be set by loadstring start_iter = -1 +start_lr = global_conf.lrate ppl_last = 100000 commands_str = "train:test" commands = {} @@ -357,6 +358,8 @@ global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str .. global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') commands = nerv.SUtil.parse_commands_set(commands_str) +global_conf.lrate = start_lr + nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) nerv.LMUtil.wait(2) os.execute("mkdir -p "..global_conf.work_dir) -- cgit v1.2.3-70-g09d2 From d2a69f2660b6a7a724605cce4d88fbbbdeec2f68 Mon Sep 17 00:00:00 2001 From: txh18 Date: Mon, 7 Dec 2015 11:37:02 +0800 Subject: ... --- nerv/examples/lmptb/bilstmlm_ptb_main.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua index 078d036..45a724b 100644 --- a/nerv/examples/lmptb/bilstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -212,7 +212,7 @@ vocab_fn = data_dir .. '/vocab' qdata_dir = root_dir .. '/ptb/questionGen/gen' global_conf = { - lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + lrate = 0.015, wcost = 1e-5, momentum = 0, clip_t = 5, cumat_type = nerv.CuMatrixFloat, mmat_type = nerv.MMatrixFloat, nn_act_default = 0, @@ -225,7 +225,7 @@ global_conf = { lr_decay = 1.003, decay_iter = 10, param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0.5", + dropout_str = "0", train_fn = train_fn, valid_fn = valid_fn, @@ -365,7 +365,7 @@ else nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) end -global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num --.. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' global_conf.param_fn = global_conf.work_dir .. "/params" -- cgit v1.2.3-70-g09d2 From 775d35373980b357ae77c9694f71d98e8ef103d6 Mon Sep 17 00:00:00 2001 From: txh18 Date: Mon, 7 Dec 2015 15:57:16 +0800 Subject: small bugs fix --- nerv/examples/lmptb/bilstmlm_ptb_main.lua | 15 ++++++--------- nerv/examples/lmptb/lmptb/lmseqreader.lua | 2 +- nerv/tnn/sutil.lua | 1 + 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua index 45a724b..b896bed 100644 --- a/nerv/examples/lmptb/bilstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -73,9 +73,6 @@ function prepare_layers(global_conf) local du = false - --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}} - --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} - local layers = { ["nerv.LSTMLayerT"] = { ["lstmFL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}, @@ -97,7 +94,7 @@ function prepare_layers(global_conf) }, ["nerv.AffineLayer"] = { - ["biAffineL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["lambda"] = {1, 1}}}, + ["biAffineL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr, ["lambda"] = {1, 1}}}, ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, }, @@ -312,10 +309,10 @@ global_conf = { else -valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' -train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' -test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' -vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' global_conf = { lrate = 0.01, wcost = 1e-5, momentum = 0, @@ -327,7 +324,7 @@ global_conf = { layer_num = 1, chunk_size = 20, batch_size = 10, - max_iter = 3, + max_iter = 2, param_random = function() return (math.random() / 5 - 0.1) end, dropout_str = "0", diff --git a/nerv/examples/lmptb/lmptb/lmseqreader.lua b/nerv/examples/lmptb/lmptb/lmseqreader.lua index 40471d5..ed791d2 100644 --- a/nerv/examples/lmptb/lmptb/lmseqreader.lua +++ b/nerv/examples/lmptb/lmptb/lmseqreader.lua @@ -179,7 +179,7 @@ function LMReader:get_batch(feeds) if got_new == false then nerv.info("lmseqreader file ends, printing stats...") - print("al_sen_start:", self.stat.al_sen_start) + nerv.printf("al_sen_start:%s\n", tostring(self.stat.al_sen_start)) return false else diff --git a/nerv/tnn/sutil.lua b/nerv/tnn/sutil.lua index 78f88c0..6a968b7 100644 --- a/nerv/tnn/sutil.lua +++ b/nerv/tnn/sutil.lua @@ -70,6 +70,7 @@ function Util.log_redirect(fn) function (fmt, ...) io.write(nerv.sprintf(fmt, ...)) nerv.log_fh:write(nerv.sprintf(fmt, ...)) + nerv.log_fh:flush() end nerv.error = function (fmt, ...) -- cgit v1.2.3-70-g09d2 From aa18efcb0b809993274bc4560b64017111dfd71c Mon Sep 17 00:00:00 2001 From: txh18 Date: Tue, 8 Dec 2015 11:14:54 +0800 Subject: small bugs fix for bilstm script --- nerv/examples/lmptb/bilstmlm_ptb_main.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua index b896bed..7e238b3 100644 --- a/nerv/examples/lmptb/bilstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -500,7 +500,7 @@ end --if commands["test"] if commands["testout"] == 1 then nerv.printf("===TEST OUT===\n") nerv.printf("q_file:\t%s\n", q_file) - local q_fn = qdata_dir .. q_file + local q_fn = qdata_dir .. '/' .. q_file global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" if final_iter ~= -1 and test_iter == -1 then test_iter = final_iter @@ -510,7 +510,8 @@ if commands["testout"] == 1 then end tnn = load_net(global_conf, test_iter) global_conf.dropout_rate = 0 - LMTrainer.lm_process_file_birnn(global_conf, q_fn, tnn, false) --false update! + LMTrainer.lm_process_file_birnn(global_conf, q_fn, tnn, false, + {["one_sen_report"] = true}) --false update! end --if commands["testout"] -- cgit v1.2.3-70-g09d2 From 2e96ce1dcfbbcedc28be96cfc1a006cd5b4d5dab Mon Sep 17 00:00:00 2001 From: txh18 Date: Wed, 9 Dec 2015 10:52:10 +0800 Subject: ... --- nerv/examples/lmptb/bilstmlm_ptb_main.lua | 4 ++-- nerv/examples/lmptb/lstmlm_ptb_main.lua | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua index 7e238b3..0472588 100644 --- a/nerv/examples/lmptb/bilstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -352,7 +352,7 @@ commands = {} test_iter = -1 --for testout(question) -local q_file = "ptb.test.txt.q10rs1_Msss.adds" +q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds" if arg[2] ~= nil then nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) @@ -500,7 +500,7 @@ end --if commands["test"] if commands["testout"] == 1 then nerv.printf("===TEST OUT===\n") nerv.printf("q_file:\t%s\n", q_file) - local q_fn = qdata_dir .. '/' .. q_file + local q_fn = q_file --qdata_dir .. '/' .. q_file global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" if final_iter ~= -1 and test_iter == -1 then test_iter = final_iter diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index b76fab8..6e3fab9 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -339,7 +339,7 @@ commands_str = "train:test" commands = {} test_iter = -1 --for testout(question) -q_file = "ptb.test.txt.q10rs1_Msss.adds" +q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds" if arg[2] ~= nil then nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) @@ -487,7 +487,7 @@ end --if commands["test"] if commands["testout"] == 1 then nerv.printf("===TEST OUT===\n") nerv.printf("q_file:\t%s\n", q_file) - local q_fn = qdata_dir .. '/' .. q_file + local q_fn = q_file --qdata_dir .. '/' .. q_file global_conf.sche_log_pre = "[SCHEDULER TESTOUT]:" if final_iter ~= -1 and test_iter == -1 then test_iter = final_iter -- cgit v1.2.3-70-g09d2 From d5c65e1df535e6ad1c827de44eb463382a07b10a Mon Sep 17 00:00:00 2001 From: txh18 Date: Wed, 9 Dec 2015 15:54:40 +0800 Subject: ... --- nerv/lib/io/chunk_file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerv/lib/io/chunk_file.c b/nerv/lib/io/chunk_file.c index 4e00b0b..71db820 100644 --- a/nerv/lib/io/chunk_file.c +++ b/nerv/lib/io/chunk_file.c @@ -112,7 +112,7 @@ static ChunkFile *open_read(const char *fn, Status *status) { for (i = 0;; offset += chunk_len, i++) { ChunkInfo *cip; - fprintf(stderr, "reading chunk %d from %d\n", i, (int)offset); + fprintf(stdout, "reading chunk %d from %d\n", i, (int)offset); /* skip to the begining of chunk i */ if (fseeko(fp, offset, SEEK_SET) != 0) { -- cgit v1.2.3-70-g09d2 From ebb8ba41886f5860f27157343bcb022eb672143c Mon Sep 17 00:00:00 2001 From: txh18 Date: Wed, 9 Dec 2015 21:31:25 +0800 Subject: ... --- nerv/examples/lmptb/lstmlm_v2_ptb_main.lua | 470 ----------------------------- 1 file changed, 470 deletions(-) delete mode 100644 nerv/examples/lmptb/lstmlm_v2_ptb_main.lua diff --git a/nerv/examples/lmptb/lstmlm_v2_ptb_main.lua b/nerv/examples/lmptb/lstmlm_v2_ptb_main.lua deleted file mode 100644 index a3d7584..0000000 --- a/nerv/examples/lmptb/lstmlm_v2_ptb_main.lua +++ /dev/null @@ -1,470 +0,0 @@ -require 'lmptb.lmvocab' -require 'lmptb.lmfeeder' -require 'lmptb.lmutil' -require 'lmptb.layer.init' ---require 'tnn.init' -require 'lmptb.lmseqreader' -require 'lm_trainer' -require 'lmptb.lstm_t_v2' - ---[[global function rename]]-- ---local printf = nerv.printf -local LMTrainer = nerv.LMTrainer ---[[global function rename ends]]-- - ---global_conf: table ---first_time: bool ---Returns: a ParamRepo -function prepare_parameters(global_conf, iter) - nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) - - global_conf.paramRepo = nerv.ParamRepo() - local paramRepo = global_conf.paramRepo - - if iter == -1 then --first time - nerv.printf("%s first time, prepare some pre-set parameters, and leaving other parameters to auto-generation...\n", global_conf.sche_log_pre) - local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') - f:close() - --[[ - ltp_ih = nerv.LinearTransParam("ltp_ih", global_conf) - ltp_ih.trans = global_conf.cumat_type(global_conf.vocab:size(), global_conf.hidden_size) --index 0 is for zero, others correspond to vocab index(starting from 1) - ltp_ih.trans:generate(global_conf.param_random) - - ltp_hh = nerv.LinearTransParam("ltp_hh", global_conf) - ltp_hh.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.hidden_size) - ltp_hh.trans:generate(global_conf.param_random) - - --ltp_ho = nerv.LinearTransParam("ltp_ho", global_conf) - --ltp_ho.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.vocab:size()) - --ltp_ho.trans:generate(global_conf.param_random) - - bp_h = nerv.BiasParam("bp_h", global_conf) - bp_h.trans = global_conf.cumat_type(1, global_conf.hidden_size) - bp_h.trans:generate(global_conf.param_random) - - --bp_o = nerv.BiasParam("bp_o", global_conf) - --bp_o.trans = global_conf.cumat_type(1, global_conf.vocab:size()) - --bp_o.trans:generate(global_conf.param_random) - - local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') - f:write_chunk(ltp_ih) - f:write_chunk(ltp_hh) - --f:write_chunk(ltp_ho) - f:write_chunk(bp_h) - --f:write_chunk(bp_o) - f:close() - ]]-- - return nil - end - - nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, global_conf.param_fn .. '.' .. tostring(iter)) - paramRepo:import({global_conf.param_fn .. '.' .. tostring(iter)}, nil, global_conf) - - nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) - - return nil -end - ---global_conf: table ---Returns: nerv.LayerRepo -function prepare_layers(global_conf) - nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) - - local pr = global_conf.paramRepo - - local du = false - - --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}} - --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} - - local layers = { - --["nerv.AffineRecurrentLayer"] = { - -- ["recurrentL1"] = recurrentLconfig, - --}, - - ["nerv.LSTMLayerTv2"] = { - ["lstmL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}, - }, - - ["nerv.DropoutLayerT"] = { - ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, - }, - - ["nerv.SelectLinearLayer"] = { - ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, - }, - - --["nerv.SigmoidLayer"] = { - -- ["sigmoidL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} - --}, - - ["nerv.CombinerLayer"] = { - ["combinerL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, - }, - - ["nerv.AffineLayer"] = { - ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, - }, - - ["nerv.SoftmaxCELayerT"] = { - ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, - }, - } - - for l = 2, global_conf.layer_num do - layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} - layers["nerv.LSTMLayerTv2"]["lstmL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}} - layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} - end - --[[ --we do not need those in the new tnn framework - printf("%s adding %d bptt layers...\n", global_conf.sche_log_pre, global_conf.bptt) - for i = 1, global_conf.bptt do - layers["nerv.IndRecurrentLayer"]["recurrentL" .. (i + 1)] = recurrentLconfig - layers["nerv.SigmoidLayer"]["sigmoidL" .. (i + 1)] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} - layers["nerv.SelectLinearLayer"]["selectL" .. (i + 1)] = {{["ltp"] = "ltp_ih"}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}}} - end - --]] - - local layerRepo = nerv.LayerRepo(layers, pr, global_conf) - nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) - return layerRepo -end - ---global_conf: table ---layerRepo: nerv.LayerRepo ---Returns: a nerv.TNN -function prepare_tnn(global_conf, layerRepo) - nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) - - --input: input_w, input_w, ... input_w_now, last_activation - local connections_t = { - {"[1]", "selectL1[1]", 0}, - - --{"selectL1[1]", "recurrentL1[1]", 0}, - --{"recurrentL1[1]", "sigmoidL1[1]", 0}, - --{"sigmoidL1[1]", "combinerL1[1]", 0}, - --{"combinerL1[1]", "recurrentL1[2]", 1}, - - {"selectL1[1]", "lstmL1[1]", 0}, - {"lstmL1[2]", "lstmL1[3]", 1}, - {"lstmL1[1]", "combinerL1[1]", 0}, - {"combinerL1[1]", "lstmL1[2]", 1}, - {"combinerL1[2]", "dropoutL1[1]", 0}, - - {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, - {"outputL[1]", "softmaxL[1]", 0}, - {"[2]", "softmaxL[2]", 0}, - {"softmaxL[1]", "[1]", 0} - } - - for l = 2, global_conf.layer_num do - table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "lstmL"..l.."[1]", 0}) - table.insert(connections_t, {"lstmL"..l.."[2]", "lstmL"..l.."[3]", 1}) - table.insert(connections_t, {"lstmL"..l.."[1]", "combinerL"..l.."[1]", 0}) - table.insert(connections_t, {"combinerL"..l.."[1]", "lstmL"..l.."[2]", 1}) - table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) - end - - --[[ - printf("%s printing DAG connections:\n", global_conf.sche_log_pre) - for key, value in pairs(connections_t) do - printf("\t%s->%s\n", key, value) - end - ]]-- - - local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, - ["dim_out"] = {1}, ["sub_layers"] = layerRepo, - ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, - }) - - tnn:init(global_conf.batch_size, global_conf.chunk_size) - - nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) - return tnn -end - -function load_net(global_conf, next_iter) - prepare_parameters(global_conf, next_iter) - local layerRepo = prepare_layers(global_conf) - local tnn = prepare_tnn(global_conf, layerRepo) - return tnn -end - -local train_fn, valid_fn, test_fn -global_conf = {} -local set = arg[1] --"test" - -if (set == "ptb") then - -root_dir = '/home/slhome/txh18/workspace' -data_dir = root_dir .. '/ptb/DATA' -train_fn = data_dir .. '/ptb.train.txt.adds' -valid_fn = data_dir .. '/ptb.valid.txt.adds' -test_fn = data_dir .. '/ptb.test.txt.adds' -vocab_fn = data_dir .. '/vocab' - -qdata_dir = root_dir .. '/ptb/questionGen/gen' - -global_conf = { - lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 300, - layer_num = 1, - chunk_size = 15, - batch_size = 20, - max_iter = 45, - lr_decay = 1.003, - decay_iter = 10, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0.5", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - vocab_fn = vocab_fn, - max_sen_len = 90, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 40000, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/ptb/EXP-nerv/lstmlm_v2.0' -} - -elseif (set == "msr_sc") then - -data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' -train_fn = data_dir .. '/normed_all.sf.len60.adds.train' -valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' -test_fn = data_dir .. '/answer_normed.adds' -vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' - -global_conf = { - lrate = 1, wcost = 1e-6, momentum = 0, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 300, - layer_num = 1, - chunk_size = 15, - batch_size = 10, - max_iter = 30, - decay_iter = 10, - lr_decay = 1.003, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - vocab_fn = vocab_fn, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 400000, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' -} - -else - -valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' -train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' -test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' -vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' - -global_conf = { - lrate = 0.01, wcost = 1e-5, momentum = 0, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 20, - layer_num = 1, - chunk_size = 2, - batch_size = 10, - max_iter = 3, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - lr_decay = 1.003, - decay_iter = 10, - vocab_fn = vocab_fn, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 10, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_lstmlm_test' -} - -end - -local lr_half = false --can not be local, to be set by loadstring -local start_iter = -1 -local ppl_last = 100000 -local commands_str = "train:test" -local commands = {} -local test_iter = -1 - ---for testout(question) -local q_file = "ptb.test.txt.q10rs1_Msss.adds" - -if arg[2] ~= nil then - nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) - loadstring(arg[2])() - nerv.LMUtil.wait(0.5) -else - nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) -end - -global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str -global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' -global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' -global_conf.param_fn = global_conf.work_dir .. "/params" -global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) -global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%X_%m_%d",os.time()) -commands = nerv.SUtil.parse_commands_set(commands_str) - -nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) -nerv.LMUtil.wait(2) -os.execute("mkdir -p "..global_conf.work_dir) -os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) - ---redirecting log outputs! -nerv.SUtil.log_redirect(global_conf.log_fn) -nerv.LMUtil.wait(2) - -----------------printing options--------------------------------- -nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) -for id, value in pairs(global_conf) do - nerv.printf("%s:\t%s\n", id, tostring(value)) -end -nerv.LMUtil.wait(2) - -nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) -nerv.printf("lr_half:\t%s\n", tostring(lr_half)) -nerv.printf("start_iter:\t%s\n", tostring(start_iter)) -nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) -nerv.printf("commds_str:\t%s\n", commands_str) -nerv.printf("test_iter:\t%s\n", tostring(test_iter)) -nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) -nerv.LMUtil.wait(2) -------------------printing options end------------------------------ - -math.randomseed(1) - -local vocab = nerv.LMVocab() -global_conf["vocab"] = vocab -nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) -global_conf.vocab:build_file(global_conf.vocab_fn, false) -ppl_rec = {} - -local final_iter = -1 -if commands["train"] == 1 then - if start_iter == -1 then - prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file - end - - if start_iter == -1 or start_iter == 0 then - nerv.printf("===INITIAL VALIDATION===\n") - local tnn = load_net(global_conf, 0) - global_conf.paramRepo = tnn:get_params() --get auto-generted params - global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file - global_conf.dropout_rate = 0 - local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! - nerv.LMUtil.wait(1) - ppl_rec[0] = {} - ppl_rec[0].valid = result:ppl_all("rnn") - ppl_last = ppl_rec[0].valid - ppl_rec[0].train = 0 - ppl_rec[0].test = 0 - ppl_rec[0].lr = 0 - - start_iter = 1 - - nerv.printf("\n") - end - - for iter = start_iter, global_conf.max_iter, 1 do - final_iter = iter --for final testing - global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" - tnn = load_net(global_conf, iter - 1) - nerv.printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) - global_conf.dropout_rate = nerv.SUtil.sche_get(global_conf.dropout_list, iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! - global_conf.dropout_rate = 0 - ppl_rec[iter] = {} - ppl_rec[iter].train = result:ppl_all("rnn") - --shuffling training file - nerv.printf("%s shuffling training file\n", global_conf.sche_log_pre) - os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) - os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) - nerv.printf("===PEEK ON TEST %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! - ppl_rec[iter].test = result:ppl_all("rnn") - nerv.printf("===VALIDATION %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! - ppl_rec[iter].valid = result:ppl_all("rnn") - ppl_rec[iter].lr = global_conf.lrate - if ((ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true) and iter > global_conf.decay_iter) then - global_conf.lrate = (global_conf.lrate * 0.6) - end - if ppl_rec[iter].valid < ppl_last then - nerv.printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) - global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) - else - nerv.printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) - os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) - end - if ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true then - lr_half = true - end - if ppl_rec[iter].valid < ppl_last then - ppl_last = ppl_rec[iter].valid - end - nerv.printf("\n") - nerv.LMUtil.wait(2) - end - nerv.info("saving final nn to param.final") - os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final') - - nerv.printf("===VALIDATION PPL record===\n") - for i, _ in pairs(ppl_rec) do - nerv.printf(" \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) - end - nerv.printf("\n") -end --if commands["train"] - -if commands["test"] == 1 then - nerv.printf("===FINAL TEST===\n") - global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" - if final_iter ~= -1 and test_iter == -1 then - test_iter = final_iter - end - if test_iter == -1 then - test_iter = "final" - end - tnn = load_net(global_conf, test_iter) - global_conf.dropout_rate = 0 - LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! -end --if commands["test"] - -if commands["testout"] == 1 then - nerv.printf("===TEST OUT===\n") - nerv.printf("q_file:\t%s\n", q_file) - local q_fn = qdata_dir .. q_file - global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" - if final_iter ~= -1 and test_iter == -1 then - test_iter = final_iter - end - if test_iter == -1 then - test_iter = "final" - end - tnn = load_net(global_conf, test_iter) - global_conf.dropout_rate = 0 - LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false) --false update! -end --if commands["testout"] - - -- cgit v1.2.3-70-g09d2