diff options
| -rw-r--r-- | nerv/examples/lmptb/lstmlm_v2_ptb_main.lua | 470 |
1 files changed, 0 insertions, 470 deletions
diff --git a/nerv/examples/lmptb/lstmlm_v2_ptb_main.lua b/nerv/examples/lmptb/lstmlm_v2_ptb_main.lua deleted file mode 100644 index a3d7584..0000000 --- a/nerv/examples/lmptb/lstmlm_v2_ptb_main.lua +++ /dev/null @@ -1,470 +0,0 @@ -require 'lmptb.lmvocab' -require 'lmptb.lmfeeder' -require 'lmptb.lmutil' -require 'lmptb.layer.init' ---require 'tnn.init' -require 'lmptb.lmseqreader' -require 'lm_trainer' -require 'lmptb.lstm_t_v2' - ---[[global function rename]]-- ---local printf = nerv.printf -local LMTrainer = nerv.LMTrainer ---[[global function rename ends]]-- - ---global_conf: table ---first_time: bool ---Returns: a ParamRepo -function prepare_parameters(global_conf, iter) - nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) - - global_conf.paramRepo = nerv.ParamRepo() - local paramRepo = global_conf.paramRepo - - if iter == -1 then --first time - nerv.printf("%s first time, prepare some pre-set parameters, and leaving other parameters to auto-generation...\n", global_conf.sche_log_pre) - local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') - f:close() - --[[ - ltp_ih = nerv.LinearTransParam("ltp_ih", global_conf) - ltp_ih.trans = global_conf.cumat_type(global_conf.vocab:size(), global_conf.hidden_size) --index 0 is for zero, others correspond to vocab index(starting from 1) - ltp_ih.trans:generate(global_conf.param_random) - - ltp_hh = nerv.LinearTransParam("ltp_hh", global_conf) - ltp_hh.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.hidden_size) - ltp_hh.trans:generate(global_conf.param_random) - - --ltp_ho = nerv.LinearTransParam("ltp_ho", global_conf) - --ltp_ho.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.vocab:size()) - --ltp_ho.trans:generate(global_conf.param_random) - - bp_h = nerv.BiasParam("bp_h", global_conf) - bp_h.trans = global_conf.cumat_type(1, global_conf.hidden_size) - bp_h.trans:generate(global_conf.param_random) - - --bp_o = nerv.BiasParam("bp_o", global_conf) - --bp_o.trans = global_conf.cumat_type(1, global_conf.vocab:size()) - --bp_o.trans:generate(global_conf.param_random) - - local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') - f:write_chunk(ltp_ih) - f:write_chunk(ltp_hh) - --f:write_chunk(ltp_ho) - f:write_chunk(bp_h) - --f:write_chunk(bp_o) - f:close() - ]]-- - return nil - end - - nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, global_conf.param_fn .. '.' .. tostring(iter)) - paramRepo:import({global_conf.param_fn .. '.' .. tostring(iter)}, nil, global_conf) - - nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) - - return nil -end - ---global_conf: table ---Returns: nerv.LayerRepo -function prepare_layers(global_conf) - nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) - - local pr = global_conf.paramRepo - - local du = false - - --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}} - --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} - - local layers = { - --["nerv.AffineRecurrentLayer"] = { - -- ["recurrentL1"] = recurrentLconfig, - --}, - - ["nerv.LSTMLayerTv2"] = { - ["lstmL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}, - }, - - ["nerv.DropoutLayerT"] = { - ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, - }, - - ["nerv.SelectLinearLayer"] = { - ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, - }, - - --["nerv.SigmoidLayer"] = { - -- ["sigmoidL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} - --}, - - ["nerv.CombinerLayer"] = { - ["combinerL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, - }, - - ["nerv.AffineLayer"] = { - ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, - }, - - ["nerv.SoftmaxCELayerT"] = { - ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, - }, - } - - for l = 2, global_conf.layer_num do - layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} - layers["nerv.LSTMLayerTv2"]["lstmL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}} - layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} - end - --[[ --we do not need those in the new tnn framework - printf("%s adding %d bptt layers...\n", global_conf.sche_log_pre, global_conf.bptt) - for i = 1, global_conf.bptt do - layers["nerv.IndRecurrentLayer"]["recurrentL" .. (i + 1)] = recurrentLconfig - layers["nerv.SigmoidLayer"]["sigmoidL" .. (i + 1)] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} - layers["nerv.SelectLinearLayer"]["selectL" .. (i + 1)] = {{["ltp"] = "ltp_ih"}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}}} - end - --]] - - local layerRepo = nerv.LayerRepo(layers, pr, global_conf) - nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) - return layerRepo -end - ---global_conf: table ---layerRepo: nerv.LayerRepo ---Returns: a nerv.TNN -function prepare_tnn(global_conf, layerRepo) - nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) - - --input: input_w, input_w, ... input_w_now, last_activation - local connections_t = { - {"<input>[1]", "selectL1[1]", 0}, - - --{"selectL1[1]", "recurrentL1[1]", 0}, - --{"recurrentL1[1]", "sigmoidL1[1]", 0}, - --{"sigmoidL1[1]", "combinerL1[1]", 0}, - --{"combinerL1[1]", "recurrentL1[2]", 1}, - - {"selectL1[1]", "lstmL1[1]", 0}, - {"lstmL1[2]", "lstmL1[3]", 1}, - {"lstmL1[1]", "combinerL1[1]", 0}, - {"combinerL1[1]", "lstmL1[2]", 1}, - {"combinerL1[2]", "dropoutL1[1]", 0}, - - {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, - {"outputL[1]", "softmaxL[1]", 0}, - {"<input>[2]", "softmaxL[2]", 0}, - {"softmaxL[1]", "<output>[1]", 0} - } - - for l = 2, global_conf.layer_num do - table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "lstmL"..l.."[1]", 0}) - table.insert(connections_t, {"lstmL"..l.."[2]", "lstmL"..l.."[3]", 1}) - table.insert(connections_t, {"lstmL"..l.."[1]", "combinerL"..l.."[1]", 0}) - table.insert(connections_t, {"combinerL"..l.."[1]", "lstmL"..l.."[2]", 1}) - table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) - end - - --[[ - printf("%s printing DAG connections:\n", global_conf.sche_log_pre) - for key, value in pairs(connections_t) do - printf("\t%s->%s\n", key, value) - end - ]]-- - - local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, - ["dim_out"] = {1}, ["sub_layers"] = layerRepo, - ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, - }) - - tnn:init(global_conf.batch_size, global_conf.chunk_size) - - nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) - return tnn -end - -function load_net(global_conf, next_iter) - prepare_parameters(global_conf, next_iter) - local layerRepo = prepare_layers(global_conf) - local tnn = prepare_tnn(global_conf, layerRepo) - return tnn -end - -local train_fn, valid_fn, test_fn -global_conf = {} -local set = arg[1] --"test" - -if (set == "ptb") then - -root_dir = '/home/slhome/txh18/workspace' -data_dir = root_dir .. '/ptb/DATA' -train_fn = data_dir .. '/ptb.train.txt.adds' -valid_fn = data_dir .. '/ptb.valid.txt.adds' -test_fn = data_dir .. '/ptb.test.txt.adds' -vocab_fn = data_dir .. '/vocab' - -qdata_dir = root_dir .. '/ptb/questionGen/gen' - -global_conf = { - lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 300, - layer_num = 1, - chunk_size = 15, - batch_size = 20, - max_iter = 45, - lr_decay = 1.003, - decay_iter = 10, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0.5", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - vocab_fn = vocab_fn, - max_sen_len = 90, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 40000, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/ptb/EXP-nerv/lstmlm_v2.0' -} - -elseif (set == "msr_sc") then - -data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' -train_fn = data_dir .. '/normed_all.sf.len60.adds.train' -valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' -test_fn = data_dir .. '/answer_normed.adds' -vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' - -global_conf = { - lrate = 1, wcost = 1e-6, momentum = 0, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 300, - layer_num = 1, - chunk_size = 15, - batch_size = 10, - max_iter = 30, - decay_iter = 10, - lr_decay = 1.003, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - vocab_fn = vocab_fn, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 400000, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' -} - -else - -valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' -train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' -test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' -vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' - -global_conf = { - lrate = 0.01, wcost = 1e-5, momentum = 0, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 20, - layer_num = 1, - chunk_size = 2, - batch_size = 10, - max_iter = 3, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - lr_decay = 1.003, - decay_iter = 10, - vocab_fn = vocab_fn, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 10, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_lstmlm_test' -} - -end - -local lr_half = false --can not be local, to be set by loadstring -local start_iter = -1 -local ppl_last = 100000 -local commands_str = "train:test" -local commands = {} -local test_iter = -1 - ---for testout(question) -local q_file = "ptb.test.txt.q10rs1_Msss.adds" - -if arg[2] ~= nil then - nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) - loadstring(arg[2])() - nerv.LMUtil.wait(0.5) -else - nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) -end - -global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str -global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' -global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' -global_conf.param_fn = global_conf.work_dir .. "/params" -global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) -global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%X_%m_%d",os.time()) -commands = nerv.SUtil.parse_commands_set(commands_str) - -nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) -nerv.LMUtil.wait(2) -os.execute("mkdir -p "..global_conf.work_dir) -os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) - ---redirecting log outputs! -nerv.SUtil.log_redirect(global_conf.log_fn) -nerv.LMUtil.wait(2) - -----------------printing options--------------------------------- -nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) -for id, value in pairs(global_conf) do - nerv.printf("%s:\t%s\n", id, tostring(value)) -end -nerv.LMUtil.wait(2) - -nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) -nerv.printf("lr_half:\t%s\n", tostring(lr_half)) -nerv.printf("start_iter:\t%s\n", tostring(start_iter)) -nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) -nerv.printf("commds_str:\t%s\n", commands_str) -nerv.printf("test_iter:\t%s\n", tostring(test_iter)) -nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) -nerv.LMUtil.wait(2) -------------------printing options end------------------------------ - -math.randomseed(1) - -local vocab = nerv.LMVocab() -global_conf["vocab"] = vocab -nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) -global_conf.vocab:build_file(global_conf.vocab_fn, false) -ppl_rec = {} - -local final_iter = -1 -if commands["train"] == 1 then - if start_iter == -1 then - prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file - end - - if start_iter == -1 or start_iter == 0 then - nerv.printf("===INITIAL VALIDATION===\n") - local tnn = load_net(global_conf, 0) - global_conf.paramRepo = tnn:get_params() --get auto-generted params - global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file - global_conf.dropout_rate = 0 - local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! - nerv.LMUtil.wait(1) - ppl_rec[0] = {} - ppl_rec[0].valid = result:ppl_all("rnn") - ppl_last = ppl_rec[0].valid - ppl_rec[0].train = 0 - ppl_rec[0].test = 0 - ppl_rec[0].lr = 0 - - start_iter = 1 - - nerv.printf("\n") - end - - for iter = start_iter, global_conf.max_iter, 1 do - final_iter = iter --for final testing - global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" - tnn = load_net(global_conf, iter - 1) - nerv.printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) - global_conf.dropout_rate = nerv.SUtil.sche_get(global_conf.dropout_list, iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! - global_conf.dropout_rate = 0 - ppl_rec[iter] = {} - ppl_rec[iter].train = result:ppl_all("rnn") - --shuffling training file - nerv.printf("%s shuffling training file\n", global_conf.sche_log_pre) - os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) - os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) - nerv.printf("===PEEK ON TEST %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! - ppl_rec[iter].test = result:ppl_all("rnn") - nerv.printf("===VALIDATION %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! - ppl_rec[iter].valid = result:ppl_all("rnn") - ppl_rec[iter].lr = global_conf.lrate - if ((ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true) and iter > global_conf.decay_iter) then - global_conf.lrate = (global_conf.lrate * 0.6) - end - if ppl_rec[iter].valid < ppl_last then - nerv.printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) - global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) - else - nerv.printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) - os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) - end - if ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true then - lr_half = true - end - if ppl_rec[iter].valid < ppl_last then - ppl_last = ppl_rec[iter].valid - end - nerv.printf("\n") - nerv.LMUtil.wait(2) - end - nerv.info("saving final nn to param.final") - os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final') - - nerv.printf("===VALIDATION PPL record===\n") - for i, _ in pairs(ppl_rec) do - nerv.printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) - end - nerv.printf("\n") -end --if commands["train"] - -if commands["test"] == 1 then - nerv.printf("===FINAL TEST===\n") - global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" - if final_iter ~= -1 and test_iter == -1 then - test_iter = final_iter - end - if test_iter == -1 then - test_iter = "final" - end - tnn = load_net(global_conf, test_iter) - global_conf.dropout_rate = 0 - LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! -end --if commands["test"] - -if commands["testout"] == 1 then - nerv.printf("===TEST OUT===\n") - nerv.printf("q_file:\t%s\n", q_file) - local q_fn = qdata_dir .. q_file - global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" - if final_iter ~= -1 and test_iter == -1 then - test_iter = final_iter - end - if test_iter == -1 then - test_iter = "final" - end - tnn = load_net(global_conf, test_iter) - global_conf.dropout_rate = 0 - LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false) --false update! -end --if commands["testout"] - - |