diff options
Diffstat (limited to 'nerv/examples/lmptb/rnnlm_ptb_main.lua')
| -rw-r--r-- | nerv/examples/lmptb/rnnlm_ptb_main.lua | 176 |
1 files changed, 102 insertions, 74 deletions
diff --git a/nerv/examples/lmptb/rnnlm_ptb_main.lua b/nerv/examples/lmptb/rnnlm_ptb_main.lua index ca62023..e2ca860 100644 --- a/nerv/examples/lmptb/rnnlm_ptb_main.lua +++ b/nerv/examples/lmptb/rnnlm_ptb_main.lua @@ -77,7 +77,7 @@ function prepare_layers(global_conf) local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} local layers = { - ["nerv.AffineRecurrentLayer"] = { + ["nerv.AffineRecurrentPlusVecLayer"] = { ["recurrentL1"] = recurrentLconfig, }, @@ -163,9 +163,11 @@ local train_fn, valid_fn, test_fn global_conf = {} local set = arg[1] --"test" +root_dir = '/home/slhome/txh18/workspace' + if (set == "ptb") then -data_dir = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata' +data_dir = root_dir .. '/nerv/nerv/nerv/examples/lmptb/PTBdata' train_fn = data_dir .. '/ptb.train.txt.adds' valid_fn = data_dir .. '/ptb.valid.txt.adds' test_fn = data_dir .. '/ptb.test.txt.adds' @@ -177,10 +179,10 @@ global_conf = { mmat_type = nerv.MMatrixFloat, nn_act_default = 0, - hidden_size = 400, --set to 400 for a stable good test PPL + hidden_size = 300, --set to 400 for a stable good test PPL chunk_size = 15, batch_size = 10, - max_iter = 35, + max_iter = 30, decay_iter = 15, param_random = function() return (math.random() / 5 - 0.1) end, @@ -191,7 +193,7 @@ global_conf = { sche_log_pre = "[SCHEDULER]:", log_w_num = 40000, --give a message when log_w_num words have been processed timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/nerv/play/ptbEXP/tnn_test' + work_dir_base = root_dir .. '/ptb/EXP-nerv/rnnlm_tnn' } elseif (set == "msr_sc") then @@ -259,6 +261,8 @@ end lr_half = false --can not be local, to be set by loadstring start_iter = -1 ppl_last = 100000 +test_iter = -1 +commands_str = "train:test" if (arg[2] ~= nil) then printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) loadstring(arg[2])() @@ -271,6 +275,9 @@ global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_si global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' global_conf.param_fn = global_conf.work_dir .. "/params" +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str .. os.date("_TT%m_%d_%X",os.time()) +global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') +commands = nerv.SUtil.parse_commands_set(commands_str) ----------------printing options--------------------------------- printf("%s printing global_conf...\n", global_conf.sche_log_pre) @@ -281,92 +288,113 @@ nerv.LMUtil.wait(2) printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) print("lr_half", lr_half) print("start_iter", start_iter) +print("test_iter", test_iter) print("ppl_last", ppl_last) printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) nerv.LMUtil.wait(2) ------------------printing options end------------------------------ -math.randomseed(1) - printf("%s creating work_dir...\n", global_conf.sche_log_pre) -os.execute("mkdir -p "..global_conf.work_dir) +os.execute("mkdir -p ".. global_conf.work_dir) os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) +--redirecting log outputs! +nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) + +math.randomseed(1) + local vocab = nerv.LMVocab() global_conf["vocab"] = vocab printf("%s building vocab...\n", global_conf.sche_log_pre) global_conf.vocab:build_file(global_conf.vocab_fn, false) ppl_rec = {} -if start_iter == -1 then - prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file -end +local final_iter = -1 +if commands["train"] == 1 then + if start_iter == -1 then + prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file + end + + if start_iter == -1 or start_iter == 0 then + print("===INITIAL VALIDATION===") + local tnn = load_net(global_conf, 0) + global_conf.paramRepo = tnn:get_params() --get auto-generted params + global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file + local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + nerv.LMUtil.wait(1) + ppl_rec[0] = {} + ppl_rec[0].valid = result:ppl_all("rnn") + ppl_last = ppl_rec[0].valid + ppl_rec[0].train = 0 + ppl_rec[0].test = 0 + ppl_rec[0].lr = 0 + + start_iter = 1 + + print() + end + + for iter = start_iter, global_conf.max_iter, 1 do + final_iter = iter --for final testing + global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" + tnn = load_net(global_conf, iter - 1) + printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! + ppl_rec[iter] = {} + ppl_rec[iter].train = result:ppl_all("rnn") + --shuffling training file + printf("%s shuffling training file\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) + os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) + printf("===PEEK ON TEST %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! + ppl_rec[iter].test = result:ppl_all("rnn") + printf("===VALIDATION %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + ppl_rec[iter].valid = result:ppl_all("rnn") + ppl_rec[iter].lr = global_conf.lrate + if ((ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true) and iter > global_conf.decay_iter) then + global_conf.lrate = (global_conf.lrate * 0.6) + end + if ppl_rec[iter].valid < ppl_last then + printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) + global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) + else + printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) + end + if ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true then + lr_half = true + end + if ppl_rec[iter].valid < ppl_last then + ppl_last = ppl_rec[iter].valid + end + printf("\n") + nerv.LMUtil.wait(2) + end -if start_iter == -1 or start_iter == 0 then - print("===INITIAL VALIDATION===") - local tnn = load_net(global_conf, 0) - global_conf.paramRepo = tnn:get_params() --get auto-generted params - global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file - local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! - nerv.LMUtil.wait(1) - ppl_rec[0] = {} - ppl_rec[0].valid = result:ppl_all("rnn") - ppl_last = ppl_rec[0].valid - ppl_rec[0].train = 0 - ppl_rec[0].test = 0 - ppl_rec[0].lr = 0 - - start_iter = 1 - - print() -end + nerv.info("saving final nn to param.final") + os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final') -local final_iter -for iter = start_iter, global_conf.max_iter, 1 do - final_iter = iter --for final testing - global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" - tnn = load_net(global_conf, iter - 1) - printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! - ppl_rec[iter] = {} - ppl_rec[iter].train = result:ppl_all("rnn") - --shuffling training file - printf("%s shuffling training file\n", global_conf.sche_log_pre) - os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) - os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) - printf("===PEEK ON TEST %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! - ppl_rec[iter].test = result:ppl_all("rnn") - printf("===VALIDATION %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! - ppl_rec[iter].valid = result:ppl_all("rnn") - ppl_rec[iter].lr = global_conf.lrate - if ((ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true) and iter > global_conf.decay_iter) then - global_conf.lrate = (global_conf.lrate * 0.6) + printf("===VALIDATION PPL record===\n") + for i, _ in pairs(ppl_rec) do + printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) end - if ppl_rec[iter].valid < ppl_last then - printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) - global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) - else - printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) - os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) - end - if ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true then - lr_half = true + printf("\n") +end --if commands["train"] + +if commands["test"] == 1 then + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter end - if ppl_rec[iter].valid < ppl_last then - ppl_last = ppl_rec[iter].valid + if test_iter == -1 then + test_iter = "final" end - printf("\n") - nerv.LMUtil.wait(2) -end -printf("===VALIDATION PPL record===\n") -for i, _ in pairs(ppl_rec) do - printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) -end -printf("\n") -printf("===FINAL TEST===\n") -global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" -tnn = load_net(global_conf, final_iter) -LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! + + printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + tnn = load_net(global_conf, test_iter) + LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["test"] |