diff options
41 files changed, 10734 insertions, 8514 deletions
diff --git a/nerv/examples/lmptb/.gitignore b/nerv/examples/lmptb/.gitignore new file mode 100644 index 0000000..55c0728 --- /dev/null +++ b/nerv/examples/lmptb/.gitignore @@ -0,0 +1 @@ +tmp_exp.sh diff --git a/nerv/examples/lmptb/bigrulm_ptb_main.lua b/nerv/examples/lmptb/bigrulm_ptb_main.lua new file mode 100644 index 0000000..3a7597f --- /dev/null +++ b/nerv/examples/lmptb/bigrulm_ptb_main.lua @@ -0,0 +1,519 @@ +require 'lmptb.lmvocab' +require 'lmptb.lmfeeder' +require 'lmptb.lmutil' +require 'lmptb.layer.init' +--require 'tnn.init' +require 'lmptb.lmseqreader' +require 'lm_trainer' + +--[[global function rename]]-- +--local printf = nerv.printf +local LMTrainer = nerv.LMTrainer +--[[global function rename ends]]-- + +--global_conf: table +--first_time: bool +--Returns: a ParamRepo +function prepare_parameters(global_conf, iter) + nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) + + global_conf.paramRepo = nerv.ParamRepo() + local paramRepo = global_conf.paramRepo + + if iter == -1 then --first time + nerv.printf("%s first time, prepare some pre-set parameters, and leaving other parameters to auto-generation...\n", global_conf.sche_log_pre) + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:close() + --[[ + ltp_ih = nerv.LinearTransParam("ltp_ih", global_conf) + ltp_ih.trans = global_conf.cumat_type(global_conf.vocab:size(), global_conf.hidden_size) --index 0 is for zero, others correspond to vocab index(starting from 1) + ltp_ih.trans:generate(global_conf.param_random) + + ltp_hh = nerv.LinearTransParam("ltp_hh", global_conf) + ltp_hh.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.hidden_size) + ltp_hh.trans:generate(global_conf.param_random) + + --ltp_ho = nerv.LinearTransParam("ltp_ho", global_conf) + --ltp_ho.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.vocab:size()) + --ltp_ho.trans:generate(global_conf.param_random) + + bp_h = nerv.BiasParam("bp_h", global_conf) + bp_h.trans = global_conf.cumat_type(1, global_conf.hidden_size) + bp_h.trans:generate(global_conf.param_random) + + --bp_o = nerv.BiasParam("bp_o", global_conf) + --bp_o.trans = global_conf.cumat_type(1, global_conf.vocab:size()) + --bp_o.trans:generate(global_conf.param_random) + + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:write_chunk(ltp_ih) + f:write_chunk(ltp_hh) + --f:write_chunk(ltp_ho) + f:write_chunk(bp_h) + --f:write_chunk(bp_o) + f:close() + ]]-- + return nil + end + + nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, global_conf.param_fn .. '.' .. tostring(iter)) + paramRepo:import({global_conf.param_fn .. '.' .. tostring(iter)}, nil, global_conf) + + nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) + + return nil +end + +--global_conf: table +--Returns: nerv.LayerRepo +function prepare_layers(global_conf) + nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) + + local pr = global_conf.paramRepo + + local du = false + + local layers = { + ["nerv.GRULayerT"] = { + ["gruFL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}}, + ["gruRL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}}, + }, + + ["nerv.DropoutLayerT"] = { + ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SelectLinearLayer"] = { + ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, + }, + + ["nerv.CombinerLayer"] = { + ["combinerXL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + ["combinerHFL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + ["combinerHRL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + }, + + ["nerv.AffineLayer"] = { + ["biAffineL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr, ["lambda"] = {1, 1}}}, + ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, + }, + + ["nerv.TanhLayer"] = { + ["biTanhL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SoftmaxCELayerT"] = { + ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, + }, + } + + if global_conf.layer_num > 1 then + nerv.error("this script currently do not support more than one layer") + end + --[[ + for l = 2, global_conf.layer_num do + layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.LSTMLayerT"]["lstmL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}} + layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} + end + ]]-- + + local layerRepo = nerv.LayerRepo(layers, pr, global_conf) + nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) + return layerRepo +end + +--global_conf: table +--layerRepo: nerv.LayerRepo +--Returns: a nerv.TNN +function prepare_tnn(global_conf, layerRepo) + nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) + + --input: input_w, input_w, ... input_w_now, last_activation + local connections_t = { + {"<input>[1]", "selectL1[1]", 0}, + + --{"selectL1[1]", "recurrentL1[1]", 0}, + --{"recurrentL1[1]", "sigmoidL1[1]", 0}, + --{"sigmoidL1[1]", "combinerL1[1]", 0}, + --{"combinerL1[1]", "recurrentL1[2]", 1}, + + {"selectL1[1]", "combinerXL1[1]", 0}, + {"combinerXL1[1]", "gruFL1[1]", 0}, + {"gruFL1[1]", "combinerHFL1[1]", 0}, + {"combinerHFL1[1]", "gruFL1[2]", 1}, + + {"combinerXL1[2]", "gruRL1[1]", 0}, + {"gruRL1[1]", "combinerHRL1[1]", 0}, + {"combinerHRL1[1]", "gruRL1[2]", -1}, + + {"combinerHFL1[2]", "biAffineL1[1]", 0}, + {"combinerHRL1[2]", "biAffineL1[2]", 0}, + {"biAffineL1[1]", "biTanhL1[1]", 0}, + {"biTanhL1[1]", "dropoutL1[1]", 0}, + + {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, + {"outputL[1]", "softmaxL[1]", 0}, + {"<input>[2]", "softmaxL[2]", 0}, + {"softmaxL[1]", "<output>[1]", 0} + } + + --[[ + for l = 2, global_conf.layer_num do + table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "lstmL"..l.."[1]", 0}) + table.insert(connections_t, {"lstmL"..l.."[2]", "lstmL"..l.."[3]", 1}) + table.insert(connections_t, {"lstmL"..l.."[1]", "combinerL"..l.."[1]", 0}) + table.insert(connections_t, {"combinerL"..l.."[1]", "lstmL"..l.."[2]", 1}) + table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) + end + ]]-- + + --[[ + printf("%s printing DAG connections:\n", global_conf.sche_log_pre) + for key, value in pairs(connections_t) do + printf("\t%s->%s\n", key, value) + end + ]]-- + + local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, + ["dim_out"] = {1}, ["sub_layers"] = layerRepo, + ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, + }) + + tnn:init(global_conf.batch_size, global_conf.chunk_size) + + nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) + return tnn +end + +function load_net(global_conf, next_iter) + prepare_parameters(global_conf, next_iter) + local layerRepo = prepare_layers(global_conf) + local tnn = prepare_tnn(global_conf, layerRepo) + return tnn +end + +local train_fn, valid_fn, test_fn +global_conf = {} +local set = arg[1] --"test" + +if (set == "ptb") then + +root_dir = '/home/slhome/txh18/workspace' +data_dir = root_dir .. '/ptb/DATA' +train_fn = data_dir .. '/ptb.train.txt.adds' +valid_fn = data_dir .. '/ptb.valid.txt.adds' +test_fn = data_dir .. '/ptb.test.txt.adds' +vocab_fn = data_dir .. '/vocab' + +qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.015, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 90, + batch_size = 32, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/ptb/EXP-nerv/bigrulm_v1.0' +} + +elseif (set == "msr_sc") then + +data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' +train_fn = data_dir .. '/normed_all.sf.len60.adds.train' +valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' +test_fn = data_dir .. '/answer_normed.adds' +vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' + +global_conf = { + lrate = 1, wcost = 1e-6, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 10, + max_iter = 30, + decay_iter = 10, + lr_decay = 1.003, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 400000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' +} + +elseif (set == "twitter") then + +root_dir = '/home/slhome/txh18/workspace' +data_dir = root_dir .. '/twitter_new/DATA' +train_fn = data_dir .. '/twitter.choose.adds' +valid_fn = data_dir .. '/twitter.valid.adds' +test_fn = data_dir .. '/comm.test.choose-ppl.adds' +vocab_fn = data_dir .. '/twitter.choose.train.vocab' + +--qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 20, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/twitter_new/EXP-nerv/bilstmlm_v1.0' +} + +else + +valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' + +global_conf = { + lrate = 0.01, wcost = 1e-5, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 20, + layer_num = 1, + chunk_size = 20, + batch_size = 10, + max_iter = 2, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + max_sen_len = 80, + lr_decay = 1.003, + decay_iter = 10, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 10, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_bilstmlm_test' +} + +end + +lr_half = false --can not be local, to be set by loadstring +start_iter = -1 +start_lr = nil +ppl_last = 100000 +commands_str = "train:test" +commands = {} +test_iter = -1 + +--for testout(question) +q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds" + +if arg[2] ~= nil then + nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) + loadstring(arg[2])() + nerv.LMUtil.wait(0.5) +else + nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) +end + +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str +global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' +global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' +global_conf.param_fn = global_conf.work_dir .. "/params" +global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) +global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') +commands = nerv.SUtil.parse_commands_set(commands_str) + +if start_lr ~= nil then + global_conf.lrate = start_lr --starting lr can be set by user(arg[2]) +end + +nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) +nerv.LMUtil.wait(2) +os.execute("mkdir -p "..global_conf.work_dir) +os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) + +--redirecting log outputs! +nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) + +----------------printing options--------------------------------- +nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) +for id, value in pairs(global_conf) do + nerv.printf("%s:\t%s\n", id, tostring(value)) +end +nerv.LMUtil.wait(2) + +nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) +nerv.printf("lr_half:\t%s\n", tostring(lr_half)) +nerv.printf("start_iter:\t%s\n", tostring(start_iter)) +nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) +nerv.printf("commands_str:\t%s\n", commands_str) +nerv.printf("test_iter:\t%s\n", tostring(test_iter)) +nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) +nerv.LMUtil.wait(2) +------------------printing options end------------------------------ + +math.randomseed(1) + +local vocab = nerv.LMVocab() +global_conf["vocab"] = vocab +nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) +global_conf.vocab:build_file(global_conf.vocab_fn, false) +ppl_rec = {} + +local final_iter = -1 +if commands["train"] == 1 then + if start_iter == -1 then + prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file + end + + if start_iter == -1 or start_iter == 0 then + nerv.printf("===INITIAL VALIDATION===\n") + local tnn = load_net(global_conf, 0) + global_conf.paramRepo = tnn:get_params() --get auto-generted params + global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file + global_conf.dropout_rate = 0 + local result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + nerv.LMUtil.wait(1) + ppl_rec[0] = {} + ppl_rec[0].valid = result:ppl_all("birnn") + ppl_last = ppl_rec[0].valid + ppl_rec[0].train = 0 + ppl_rec[0].test = 0 + ppl_rec[0].lr = 0 + + start_iter = 1 + + nerv.printf("\n") + end + + for iter = start_iter, global_conf.max_iter, 1 do + final_iter = iter --for final testing + global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" + tnn = load_net(global_conf, iter - 1) + nerv.printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) + global_conf.dropout_rate = nerv.SUtil.sche_get(global_conf.dropout_list, iter) + result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! + global_conf.dropout_rate = 0 + ppl_rec[iter] = {} + ppl_rec[iter].train = result:ppl_all("birnn") + --shuffling training file + nerv.printf("%s shuffling training file\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) + os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) + nerv.printf("===PEEK ON TEST %d===\n", iter) + result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.test_fn, tnn, false) --false update! + ppl_rec[iter].test = result:ppl_all("birnn") + nerv.printf("===VALIDATION %d===\n", iter) + result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + ppl_rec[iter].valid = result:ppl_all("birnn") + ppl_rec[iter].lr = global_conf.lrate + if ((ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true) and iter > global_conf.decay_iter) then + global_conf.lrate = (global_conf.lrate * 0.6) + end + if ppl_rec[iter].valid < ppl_last then + nerv.printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) + global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) + else + nerv.printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) + end + if ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true then + lr_half = true + end + if ppl_rec[iter].valid < ppl_last then + ppl_last = ppl_rec[iter].valid + end + nerv.printf("\n") + nerv.LMUtil.wait(2) + end + nerv.info("saving final nn to param.final") + os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final') + + nerv.printf("===VALIDATION PPL record===\n") + for i, _ in pairs(ppl_rec) do + nerv.printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) + end + nerv.printf("\n") +end --if commands["train"] + +if commands["test"] == 1 then + nerv.printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_birnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["test"] + +if commands["testout"] == 1 then + nerv.printf("===TEST OUT===\n") + nerv.printf("q_file:\t%s\n", q_file) + local q_fn = q_file --qdata_dir .. '/' .. q_file + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_birnn(global_conf, q_fn, tnn, false, + {["one_sen_report"] = true}) --false update! +end --if commands["testout"] + + diff --git a/nerv/examples/lmptb/bilstmlm_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_ptb_main.lua index 0472588..e88eea2 100644 --- a/nerv/examples/lmptb/bilstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/bilstmlm_ptb_main.lua @@ -345,7 +345,7 @@ end lr_half = false --can not be local, to be set by loadstring start_iter = -1 -start_lr = global_conf.lrate +start_lr = nil ppl_last = 100000 commands_str = "train:test" commands = {} @@ -371,7 +371,9 @@ global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str .. global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') commands = nerv.SUtil.parse_commands_set(commands_str) -global_conf.lrate = start_lr --starting lr can be set by user(arg[2]) +if start_lr ~= nil then + global_conf.lrate = start_lr --starting lr can be set by user(arg[2]) +end nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) nerv.LMUtil.wait(2) diff --git a/nerv/examples/lmptb/bilstmlm_v2_ptb_main.lua b/nerv/examples/lmptb/bilstmlm_v2_ptb_main.lua new file mode 100644 index 0000000..4f52f29 --- /dev/null +++ b/nerv/examples/lmptb/bilstmlm_v2_ptb_main.lua @@ -0,0 +1,522 @@ +--[[ +The bilstm_v2 slightly changed the structure of tnn so that the current prediction won't have info about the very word to predict, so we should not get an amazingly low PPL +]]-- +require 'lmptb.lmvocab' +require 'lmptb.lmfeeder' +require 'lmptb.lmutil' +require 'lmptb.layer.init' +--require 'tnn.init' +require 'lmptb.lmseqreader' +require 'lm_trainer' + +--[[global function rename]]-- +--local printf = nerv.printf +local LMTrainer = nerv.LMTrainer +--[[global function rename ends]]-- + +--global_conf: table +--first_time: bool +--Returns: a ParamRepo +function prepare_parameters(global_conf, iter) + nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) + + global_conf.paramRepo = nerv.ParamRepo() + local paramRepo = global_conf.paramRepo + + if iter == -1 then --first time + nerv.printf("%s first time, prepare some pre-set parameters, and leaving other parameters to auto-generation...\n", global_conf.sche_log_pre) + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:close() + --[[ + ltp_ih = nerv.LinearTransParam("ltp_ih", global_conf) + ltp_ih.trans = global_conf.cumat_type(global_conf.vocab:size(), global_conf.hidden_size) --index 0 is for zero, others correspond to vocab index(starting from 1) + ltp_ih.trans:generate(global_conf.param_random) + + ltp_hh = nerv.LinearTransParam("ltp_hh", global_conf) + ltp_hh.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.hidden_size) + ltp_hh.trans:generate(global_conf.param_random) + + --ltp_ho = nerv.LinearTransParam("ltp_ho", global_conf) + --ltp_ho.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.vocab:size()) + --ltp_ho.trans:generate(global_conf.param_random) + + bp_h = nerv.BiasParam("bp_h", global_conf) + bp_h.trans = global_conf.cumat_type(1, global_conf.hidden_size) + bp_h.trans:generate(global_conf.param_random) + + --bp_o = nerv.BiasParam("bp_o", global_conf) + --bp_o.trans = global_conf.cumat_type(1, global_conf.vocab:size()) + --bp_o.trans:generate(global_conf.param_random) + + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:write_chunk(ltp_ih) + f:write_chunk(ltp_hh) + --f:write_chunk(ltp_ho) + f:write_chunk(bp_h) + --f:write_chunk(bp_o) + f:close() + ]]-- + return nil + end + + nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, global_conf.param_fn .. '.' .. tostring(iter)) + paramRepo:import({global_conf.param_fn .. '.' .. tostring(iter)}, nil, global_conf) + + nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) + + return nil +end + +--global_conf: table +--Returns: nerv.LayerRepo +function prepare_layers(global_conf) + nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) + + local pr = global_conf.paramRepo + + local du = false + + local layers = { + ["nerv.LSTMLayerT"] = { + ["lstmFL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}, + ["lstmRL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}, + }, + + ["nerv.DropoutLayerT"] = { + ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SelectLinearLayer"] = { + ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, + }, + + ["nerv.CombinerLayer"] = { + ["combinerXL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + ["combinerHFL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + ["combinerHRL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + }, + + ["nerv.AffineLayer"] = { + ["biAffineL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr, ["lambda"] = {1, 1}}}, + ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, + }, + + ["nerv.TanhLayer"] = { + ["biTanhL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SoftmaxCELayerT"] = { + ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, + }, + } + + if global_conf.layer_num > 1 then + nerv.error("this script currently do not support more than one layer") + end + --[[ + for l = 2, global_conf.layer_num do + layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.LSTMLayerT"]["lstmL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}} + layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} + end + ]]-- + + local layerRepo = nerv.LayerRepo(layers, pr, global_conf) + nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) + return layerRepo +end + +--global_conf: table +--layerRepo: nerv.LayerRepo +--Returns: a nerv.TNN +function prepare_tnn(global_conf, layerRepo) + nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) + + --input: input_w, input_w, ... input_w_now, last_activation + local connections_t = { + {"<input>[1]", "selectL1[1]", 0}, + + --{"selectL1[1]", "recurrentL1[1]", 0}, + --{"recurrentL1[1]", "sigmoidL1[1]", 0}, + --{"sigmoidL1[1]", "combinerL1[1]", 0}, + --{"combinerL1[1]", "recurrentL1[2]", 1}, + + {"selectL1[1]", "combinerXL1[1]", 0}, + {"combinerXL1[1]", "lstmFL1[1]", 0}, + {"lstmFL1[1]", "combinerHFL1[1]", 0}, + {"combinerHFL1[1]", "lstmFL1[2]", 1}, + {"lstmFL1[2]", "lstmFL1[3]", 1}, + {"combinerXL1[2]", "lstmRL1[1]", 0}, + {"lstmRL1[1]", "combinerHRL1[1]", 0}, + {"combinerHRL1[1]", "lstmRL1[2]", -1}, + {"lstmRL1[2]", "lstmRL1[3]", -1}, + {"combinerHFL1[2]", "biAffineL1[1]", 0}, + {"combinerHRL1[2]", "biAffineL1[2]", -2}, + {"biAffineL1[1]", "biTanhL1[1]", 0}, + {"biTanhL1[1]", "dropoutL1[1]", 0}, + + {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, + {"outputL[1]", "softmaxL[1]", 0}, + {"<input>[2]", "softmaxL[2]", 0}, + {"softmaxL[1]", "<output>[1]", 0} + } + + --[[ + for l = 2, global_conf.layer_num do + table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "lstmL"..l.."[1]", 0}) + table.insert(connections_t, {"lstmL"..l.."[2]", "lstmL"..l.."[3]", 1}) + table.insert(connections_t, {"lstmL"..l.."[1]", "combinerL"..l.."[1]", 0}) + table.insert(connections_t, {"combinerL"..l.."[1]", "lstmL"..l.."[2]", 1}) + table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) + end + ]]-- + + --[[ + printf("%s printing DAG connections:\n", global_conf.sche_log_pre) + for key, value in pairs(connections_t) do + printf("\t%s->%s\n", key, value) + end + ]]-- + + local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, + ["dim_out"] = {1}, ["sub_layers"] = layerRepo, + ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, + }) + + tnn:init(global_conf.batch_size, global_conf.chunk_size) + + nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) + return tnn +end + +function load_net(global_conf, next_iter) + prepare_parameters(global_conf, next_iter) + local layerRepo = prepare_layers(global_conf) + local tnn = prepare_tnn(global_conf, layerRepo) + return tnn +end + +local train_fn, valid_fn, test_fn +global_conf = {} +local set = arg[1] --"test" + +if (set == "ptb") then + +root_dir = '/home/slhome/txh18/workspace' +data_dir = root_dir .. '/ptb/DATA' +train_fn = data_dir .. '/ptb.train.txt.adds' +valid_fn = data_dir .. '/ptb.valid.txt.adds' +test_fn = data_dir .. '/ptb.test.txt.adds' +vocab_fn = data_dir .. '/vocab' + +qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.015, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 90, + batch_size = 20, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/ptb/EXP-nerv/bilstmlm_v2.0' +} + +elseif (set == "msr_sc") then + +data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' +train_fn = data_dir .. '/normed_all.sf.len60.adds.train' +valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' +test_fn = data_dir .. '/answer_normed.adds' +vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' + +global_conf = { + lrate = 1, wcost = 1e-6, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 10, + max_iter = 30, + decay_iter = 10, + lr_decay = 1.003, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 400000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' +} + +elseif (set == "twitter") then + +root_dir = '/home/slhome/txh18/workspace' +data_dir = root_dir .. '/twitter_new/DATA' +train_fn = data_dir .. '/twitter.choose.adds' +valid_fn = data_dir .. '/twitter.valid.adds' +test_fn = data_dir .. '/comm.test.choose-ppl.adds' +vocab_fn = data_dir .. '/twitter.choose.train.vocab' + +--qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 20, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/twitter_new/EXP-nerv/bilstmlm_v1.0' +} + +else + +valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' +vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text' + +global_conf = { + lrate = 0.01, wcost = 1e-5, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 20, + layer_num = 1, + chunk_size = 20, + batch_size = 10, + max_iter = 2, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + max_sen_len = 80, + lr_decay = 1.003, + decay_iter = 10, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 10, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_bilstmlm_test' +} + +end + +lr_half = false --can not be local, to be set by loadstring +start_iter = -1 +ppl_last = 100000 +commands_str = "train:test" +commands = {} +test_iter = -1 +start_lr = nil + +--for testout(question) +q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds" + +if arg[2] ~= nil then + nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) + loadstring(arg[2])() + nerv.LMUtil.wait(0.5) +else + nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) +end + + +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num --.. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str +global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' +global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' +global_conf.param_fn = global_conf.work_dir .. "/params" +global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) +global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') +commands = nerv.SUtil.parse_commands_set(commands_str) +if start_lr ~= nil then + global_conf.lrate = start_lr --starting lr can be set by user(arg[2]) +end + +nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) +nerv.LMUtil.wait(2) +os.execute("mkdir -p "..global_conf.work_dir) +os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) + +--redirecting log outputs! +nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) + +----------------printing options--------------------------------- +nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) +for id, value in pairs(global_conf) do + nerv.printf("%s:\t%s\n", id, tostring(value)) +end +nerv.LMUtil.wait(2) + +nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) +nerv.printf("lr_half:\t%s\n", tostring(lr_half)) +nerv.printf("start_iter:\t%s\n", tostring(start_iter)) +nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) +nerv.printf("commands_str:\t%s\n", commands_str) +nerv.printf("test_iter:\t%s\n", tostring(test_iter)) +nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) +nerv.LMUtil.wait(2) +------------------printing options end------------------------------ + +math.randomseed(1) + +local vocab = nerv.LMVocab() +global_conf["vocab"] = vocab +nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) +global_conf.vocab:build_file(global_conf.vocab_fn, false) +ppl_rec = {} + +local final_iter = -1 +if commands["train"] == 1 then + if start_iter == -1 then + prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file + end + + if start_iter == -1 or start_iter == 0 then + nerv.printf("===INITIAL VALIDATION===\n") + local tnn = load_net(global_conf, 0) + global_conf.paramRepo = tnn:get_params() --get auto-generted params + global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file + global_conf.dropout_rate = 0 + local result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + nerv.LMUtil.wait(1) + ppl_rec[0] = {} + ppl_rec[0].valid = result:ppl_all("birnn") + ppl_last = ppl_rec[0].valid + ppl_rec[0].train = 0 + ppl_rec[0].test = 0 + ppl_rec[0].lr = 0 + + start_iter = 1 + + nerv.printf("\n") + end + + for iter = start_iter, global_conf.max_iter, 1 do + final_iter = iter --for final testing + global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" + tnn = load_net(global_conf, iter - 1) + nerv.printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) + global_conf.dropout_rate = nerv.SUtil.sche_get(global_conf.dropout_list, iter) + result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! + global_conf.dropout_rate = 0 + ppl_rec[iter] = {} + ppl_rec[iter].train = result:ppl_all("birnn") + --shuffling training file + nerv.printf("%s shuffling training file\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) + os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) + nerv.printf("===PEEK ON TEST %d===\n", iter) + result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.test_fn, tnn, false) --false update! + ppl_rec[iter].test = result:ppl_all("birnn") + nerv.printf("===VALIDATION %d===\n", iter) + result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + ppl_rec[iter].valid = result:ppl_all("birnn") + ppl_rec[iter].lr = global_conf.lrate + if ((ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true) and iter > global_conf.decay_iter) then + global_conf.lrate = (global_conf.lrate * 0.6) + end + if ppl_rec[iter].valid < ppl_last then + nerv.printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) + global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) + else + nerv.printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) + end + if ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true then + lr_half = true + end + if ppl_rec[iter].valid < ppl_last then + ppl_last = ppl_rec[iter].valid + end + nerv.printf("\n") + nerv.LMUtil.wait(2) + end + nerv.info("saving final nn to param.final") + os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final') + + nerv.printf("===VALIDATION PPL record===\n") + for i, _ in pairs(ppl_rec) do + nerv.printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) + end + nerv.printf("\n") +end --if commands["train"] + +if commands["test"] == 1 then + nerv.printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_birnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["test"] + +if commands["testout"] == 1 then + nerv.printf("===TEST OUT===\n") + nerv.printf("q_file:\t%s\n", q_file) + local q_fn = q_file --qdata_dir .. '/' .. q_file + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_birnn(global_conf, q_fn, tnn, false, + {["one_sen_report"] = true}) --false update! +end --if commands["testout"] + + diff --git a/nerv/examples/lmptb/grulm_ptb_main.lua b/nerv/examples/lmptb/grulm_ptb_main.lua new file mode 100644 index 0000000..ef5d7f9 --- /dev/null +++ b/nerv/examples/lmptb/grulm_ptb_main.lua @@ -0,0 +1,517 @@ +require 'lmptb.lmvocab' +require 'lmptb.lmfeeder' +require 'lmptb.lmutil' +require 'lmptb.layer.init' +--require 'tnn.init' +require 'lmptb.lmseqreader' +require 'lm_trainer' + +--[[global function rename]]-- +--local printf = nerv.printf +local LMTrainer = nerv.LMTrainer +--[[global function rename ends]]-- + +--global_conf: table +--first_time: bool +--Returns: a ParamRepo +function prepare_parameters(global_conf, iter) + nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) + + global_conf.paramRepo = nerv.ParamRepo() + local paramRepo = global_conf.paramRepo + + if iter == -1 then --first time + nerv.printf("%s first time, prepare some pre-set parameters, and leaving other parameters to auto-generation...\n", global_conf.sche_log_pre) + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:close() + --[[ + ltp_ih = nerv.LinearTransParam("ltp_ih", global_conf) + ltp_ih.trans = global_conf.cumat_type(global_conf.vocab:size(), global_conf.hidden_size) --index 0 is for zero, others correspond to vocab index(starting from 1) + ltp_ih.trans:generate(global_conf.param_random) + + ltp_hh = nerv.LinearTransParam("ltp_hh", global_conf) + ltp_hh.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.hidden_size) + ltp_hh.trans:generate(global_conf.param_random) + + --ltp_ho = nerv.LinearTransParam("ltp_ho", global_conf) + --ltp_ho.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.vocab:size()) + --ltp_ho.trans:generate(global_conf.param_random) + + bp_h = nerv.BiasParam("bp_h", global_conf) + bp_h.trans = global_conf.cumat_type(1, global_conf.hidden_size) + bp_h.trans:generate(global_conf.param_random) + + --bp_o = nerv.BiasParam("bp_o", global_conf) + --bp_o.trans = global_conf.cumat_type(1, global_conf.vocab:size()) + --bp_o.trans:generate(global_conf.param_random) + + local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w') + f:write_chunk(ltp_ih) + f:write_chunk(ltp_hh) + --f:write_chunk(ltp_ho) + f:write_chunk(bp_h) + --f:write_chunk(bp_o) + f:close() + ]]-- + return nil + end + + nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, global_conf.param_fn .. '.' .. tostring(iter)) + paramRepo:import({global_conf.param_fn .. '.' .. tostring(iter)}, nil, global_conf) + + nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) + + return nil +end + +--global_conf: table +--Returns: nerv.LayerRepo +function prepare_layers(global_conf) + nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) + + local pr = global_conf.paramRepo + + local du = false + + --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}} + --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} + + local layers = { + ["nerv.GRULayerT"] = { + ["gruL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}}, + }, + + ["nerv.DropoutLayerT"] = { + ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SelectLinearLayer"] = { + ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, + }, + + ["nerv.CombinerLayer"] = { + ["combinerL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + }, + + ["nerv.AffineLayer"] = { + ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, + }, + + ["nerv.SoftmaxCELayerT"] = { + ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, + }, + } + + for l = 2, global_conf.layer_num do + layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.GRULayerT"]["gruL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}} + layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} + end + --[[ --we do not need those in the new tnn framework + printf("%s adding %d bptt layers...\n", global_conf.sche_log_pre, global_conf.bptt) + for i = 1, global_conf.bptt do + layers["nerv.IndRecurrentLayer"]["recurrentL" .. (i + 1)] = recurrentLconfig + layers["nerv.SigmoidLayer"]["sigmoidL" .. (i + 1)] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.SelectLinearLayer"]["selectL" .. (i + 1)] = {{["ltp"] = "ltp_ih"}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}}} + end + --]] + + local layerRepo = nerv.LayerRepo(layers, pr, global_conf) + nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) + return layerRepo +end + +--global_conf: table +--layerRepo: nerv.LayerRepo +--Returns: a nerv.TNN +function prepare_tnn(global_conf, layerRepo) + nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) + + --input: input_w, input_w, ... input_w_now, last_activation + local connections_t = { + {"<input>[1]", "selectL1[1]", 0}, + + --{"selectL1[1]", "recurrentL1[1]", 0}, + --{"recurrentL1[1]", "sigmoidL1[1]", 0}, + --{"sigmoidL1[1]", "combinerL1[1]", 0}, + --{"combinerL1[1]", "recurrentL1[2]", 1}, + + {"selectL1[1]", "gruL1[1]", 0}, + {"gruL1[1]", "combinerL1[1]", 0}, + {"combinerL1[1]", "gruL1[2]", 1}, + {"combinerL1[2]", "dropoutL1[1]", 0}, + + {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, + {"outputL[1]", "softmaxL[1]", 0}, + {"<input>[2]", "softmaxL[2]", 0}, + {"softmaxL[1]", "<output>[1]", 0} + } + + for l = 2, global_conf.layer_num do + table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "gruL"..l.."[1]", 0}) + table.insert(connections_t, {"gruL"..l.."[1]", "combinerL"..l.."[1]", 0}) + table.insert(connections_t, {"combinerL"..l.."[1]", "gruL"..l.."[2]", 1}) + table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) + end + + --[[ + printf("%s printing DAG connections:\n", global_conf.sche_log_pre) + for key, value in pairs(connections_t) do + printf("\t%s->%s\n", key, value) + end + ]]-- + + local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, + ["dim_out"] = {1}, ["sub_layers"] = layerRepo, + ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, + }) + + tnn:init(global_conf.batch_size, global_conf.chunk_size) + + nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) + return tnn +end + +function load_net(global_conf, next_iter) + prepare_parameters(global_conf, next_iter) + local layerRepo = prepare_layers(global_conf) + local tnn = prepare_tnn(global_conf, layerRepo) + return tnn +end + +local train_fn, valid_fn, test_fn +global_conf = {} +local set = arg[1] --"test" + +root_dir = '/home/slhome/txh18/workspace' + +if (set == "ptb") then + +data_dir = root_dir .. '/ptb/DATA' +train_fn = data_dir .. '/ptb.train.txt.adds' +valid_fn = data_dir .. '/ptb.valid.txt.adds' +test_fn = data_dir .. '/ptb.test.txt.adds' +vocab_fn = data_dir .. '/vocab' + +qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 32, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0.5", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/ptb/EXP-nerv/grulm_v1.0' +} + +elseif (set == "msr_sc") then + +data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' +train_fn = data_dir .. '/normed_all.sf.len60.adds.train' +valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' +test_fn = data_dir .. '/answer_normed.adds' +vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' + +global_conf = { + lrate = 1, wcost = 1e-6, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 10, + max_iter = 30, + decay_iter = 10, + lr_decay = 1.003, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 400000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' +} + +elseif (set == "twitter") then + +data_dir = root_dir .. '/twitter_new/DATA' +train_fn = data_dir .. '/twitter.choose2.adds' +valid_fn = data_dir .. '/twitter.valid.adds' +test_fn = data_dir .. '/comm.test.choose-ppl.adds' +vocab_fn = data_dir .. '/twitter.choose.train.vocab' + +--qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 32, + max_iter = 30, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0.5", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 32, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/twitter_new/EXP-nerv/grulm_v1.0' +} + +else + +valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' + +global_conf = { + lrate = 0.01, wcost = 1e-5, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 20, + layer_num = 1, + chunk_size = 2, + batch_size = 10, + max_iter = 3, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + max_sen_len = 80, + lr_decay = 1.003, + decay_iter = 10, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 10, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_lstmlm_test' +} + +end + +lr_half = false --can not be local, to be set by loadstring +start_iter = -1 +start_lr = nil +ppl_last = 100000 +commands_str = "train:test" +commands = {} +test_iter = -1 +--for testout(question) +q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds" + +if arg[2] ~= nil then + nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) + loadstring(arg[2])() + nerv.LMUtil.wait(0.5) +else + nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) +end + +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str +global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' +global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' +global_conf.param_fn = global_conf.work_dir .. "/params" +global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) +global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') +commands = nerv.SUtil.parse_commands_set(commands_str) + +if start_lr ~= nil then + global_conf.lrate = start_lr +end + +nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) +nerv.LMUtil.wait(2) +os.execute("mkdir -p "..global_conf.work_dir) +os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) + +--redirecting log outputs! +nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) + +----------------printing options--------------------------------- +nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) +for id, value in pairs(global_conf) do + nerv.printf("%s:\t%s\n", id, tostring(value)) +end +nerv.LMUtil.wait(2) + +nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) +nerv.printf("lr_half:\t%s\n", tostring(lr_half)) +nerv.printf("start_iter:\t%s\n", tostring(start_iter)) +nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) +nerv.printf("commands_str:\t%s\n", commands_str) +nerv.printf("test_iter:\t%s\n", tostring(test_iter)) +nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) +nerv.LMUtil.wait(2) +------------------printing options end------------------------------ + +math.randomseed(1) + +local vocab = nerv.LMVocab() +global_conf["vocab"] = vocab +nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) +global_conf.vocab:build_file(global_conf.vocab_fn, false) +ppl_rec = {} + +local final_iter = -1 +if commands["train"] == 1 then + if start_iter == -1 then + prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file + end + + if start_iter == -1 or start_iter == 0 then + nerv.printf("===INITIAL VALIDATION===\n") + local tnn = load_net(global_conf, 0) + global_conf.paramRepo = tnn:get_params() --get auto-generted params + global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file + global_conf.dropout_rate = 0 + local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + nerv.LMUtil.wait(1) + ppl_rec[0] = {} + ppl_rec[0].valid = result:ppl_all("rnn") + ppl_last = ppl_rec[0].valid + ppl_rec[0].train = 0 + ppl_rec[0].test = 0 + ppl_rec[0].lr = 0 + + start_iter = 1 + + nerv.printf("\n") + end + + for iter = start_iter, global_conf.max_iter, 1 do + final_iter = iter --for final testing + global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" + tnn = load_net(global_conf, iter - 1) + nerv.printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) + global_conf.dropout_rate = nerv.SUtil.sche_get(global_conf.dropout_list, iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! + global_conf.dropout_rate = 0 + ppl_rec[iter] = {} + ppl_rec[iter].train = result:ppl_all("rnn") + --shuffling training file + nerv.printf("%s shuffling training file\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) + os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) + nerv.printf("===PEEK ON TEST %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! + ppl_rec[iter].test = result:ppl_all("rnn") + nerv.printf("===VALIDATION %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + ppl_rec[iter].valid = result:ppl_all("rnn") + ppl_rec[iter].lr = global_conf.lrate + if ((ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true) and iter > global_conf.decay_iter) then + global_conf.lrate = (global_conf.lrate * 0.6) + end + if ppl_rec[iter].valid < ppl_last then + nerv.printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) + global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) + else + nerv.printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) + end + if ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true then + lr_half = true + end + if ppl_rec[iter].valid < ppl_last then + ppl_last = ppl_rec[iter].valid + end + nerv.printf("\n") + nerv.LMUtil.wait(2) + end + nerv.info("saving final nn to param.final") + os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final') + + nerv.printf("===VALIDATION PPL record===\n") + for i, _ in pairs(ppl_rec) do + nerv.printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) + end + nerv.printf("\n") +end --if commands["train"] + +if commands["test"] == 1 then + nerv.printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["test"] + +if commands["testout"] == 1 then + nerv.printf("===TEST OUT===\n") + nerv.printf("q_file:\t%s\n", q_file) + local q_fn = q_file --qdata_dir .. '/' .. q_file + global_conf.sche_log_pre = "[SCHEDULER TESTOUT]:" + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + tnn = load_net(global_conf, test_iter) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false, + {["one_sen_report"] = true}) --false update! +end --if commands["testout"] + +if commands["wordprob"] == 1 then + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + + nerv.printf("===WORD PROB===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + tnn = load_net(global_conf, test_iter) + LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false, {["word_prob_report"] = true}) --false update! +end --if commands["wordprob"] + diff --git a/nerv/examples/lmptb/lm_sampler.lua b/nerv/examples/lmptb/lm_sampler.lua new file mode 100644 index 0000000..c25a75c --- /dev/null +++ b/nerv/examples/lmptb/lm_sampler.lua @@ -0,0 +1,104 @@ +local LMSampler = nerv.class('nerv.LMSampler') + +function LMSampler:__init(global_conf) + self.log_pre = "LMSampler" + self.gconf = global_conf + self.vocab = self.gconf.vocab + self.sen_end_token = self.vocab.sen_end_token + self.sen_end_id = self.vocab:get_word_str(self.sen_end_token).id +end + +function LMSampler:load_dagL(dagL) + self.batch_size = self.gconf.batch_size + self.chunk_size = self.gconf.chunk_size + + nerv.printf("%s loading dagL\n", self.log_pre) + + self.dagL = dagL + + self.dagL_inputs = {} + self.dagL_inputs[1] = global_conf.cumat_type(global_conf.batch_size, 1) + self.dagL_inputs[1]:fill(self.sen_end_id - 1) + self.dagL_inputs[2] = global_conf.cumat_type(global_conf.batch_size, global_conf.hidden_size) + self.dagL_inputs[2]:fill(0) + + self.dagL_outputs = {} + self.dagL_outputs[1] = global_conf.cumat_type(global_conf.batch_size, global_conf.vocab:size()) + self.dagL_outputs[2] = global_conf.cumat_type(global_conf.batch_size, global_conf.hidden_size) + + self.smout_d = global_conf.cumat_type(self.batch_size, self.vocab:size()) + self.smout_h = global_conf.mmat_type(self.batch_size, self.vocab:size()) + + self.store = {} + for i = 1, self.batch_size do + self.store[i] = {} + self.store[i][1] = {} + self.store[i][1].w = self.sen_end_token + self.store[i][1].id = self.sen_end_id + self.store[i][1].p = 0 + end + self.repo = {} +end + +function LMSampler:sample_to_store(smout) + for i = 1, self.batch_size do + local ran = math.random() + local s = 0 + local id = self.vocab:size() + for j = 0, self.vocab:size() - 1 do + s = s + smout[i - 1][j] + if s >= ran then + id = j + 1 + break + end + end + if #self.store[i] >= self.chunk_size - 2 then + id = self.sen_end_id + end + local tmp = {} + tmp.w = self.vocab:get_word_id(id).str + tmp.id = id + tmp.p = smout[i - 1][id - 1] + table.insert(self.store[i], tmp) + end +end + +--Returns: LMResult +function LMSampler:lm_sample_rnn_dagL(sample_num, p_conf) + local dagL = self.dagL + local inputs = self.dagL_inputs + local outputs = self.dagL_outputs + + while #self.repo < sample_num do + dagL:propagate(inputs, outputs) + inputs[2]:copy_fromd(outputs[2]) --copy hidden activation + + self.smout_d:softmax(outputs[1]) + self.smout_d:copy_toh(self.smout_h) + + self:sample_to_store(self.smout_h) + for i = 1, self.batch_size do + inputs[1][i - 1][0] = self.store[i][#self.store[i]].id - 1 + if self.store[i][#self.store[i]].id == self.sen_end_id then --meet a sentence end + if #self.store[i] >= 3 then + self.repo[#self.repo + 1] = self.store[i] + end + inputs[2][i - 1]:fill(0) + self.store[i] = {} + self.store[i][1] = {} + self.store[i][1].w = self.sen_end_token + self.store[i][1].id = self.sen_end_id + self.store[i][1].p = 0 + end + end + + collectgarbage("collect") + end + + local res = {} + for i = 1, sample_num do + res[i] = self.repo[#self.repo] + self.repo[#self.repo] = nil + end + return res +end diff --git a/nerv/examples/lmptb/lm_trainer.lua b/nerv/examples/lmptb/lm_trainer.lua index 3b8b5c3..8a9744b 100644 --- a/nerv/examples/lmptb/lm_trainer.lua +++ b/nerv/examples/lmptb/lm_trainer.lua @@ -23,14 +23,16 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) end local reader local r_conf = {} + if p_conf.compressed_label ~= nil then + r_conf.compressed_label = p_conf.compressed_label + end local chunk_size, batch_size - if p_conf.one_sen_report == true then --report log prob one by one sentence + if p_conf.one_sen_report == true or p_conf.word_prob_report == true then --report log prob one by one sentence if do_train == true then - nerv.warning("LMTrainer.lm_process_file_rnn: warning, one_sen_report is true while do_train is also true, strange") + nerv.error("LMTrainer.lm_process_file_rnn: warning, one_sen_report(or word_prob_report) is true while do_train is also true, strange") end - nerv.printf("lm_process_file_rnn: one_sen report mode, set batch_size to 1 and chunk_size to max_sen_len(%d)\n", - global_conf.max_sen_len) - batch_size = 1 + nerv.printf("lm_process_file_rnn: one_sen(or word_prob) report mode, set chunk_size to max_sen_len(%d)\n", global_conf.max_sen_len) + batch_size = global_conf.batch_size chunk_size = global_conf.max_sen_len r_conf["se_mode"] = true else @@ -94,12 +96,15 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) global_conf.timer:tic('tnn_afterprocess') local sen_logp = {} + local word_prob = {} for t = 1, chunk_size, 1 do + word_prob[t] = {} tnn.outputs_m[t][1]:copy_toh(neto_bakm) for i = 1, batch_size, 1 do if (feeds.labels_s[t][i] ~= global_conf.vocab.null_token) then --result:add("rnn", feeds.labels_s[t][i], math.exp(tnn.outputs_m[t][1][i - 1][0])) result:add("rnn", feeds.labels_s[t][i], math.exp(neto_bakm[i - 1][0])) + word_prob[t][i] = math.exp(neto_bakm[i - 1][0]) if sen_logp[i] == nil then sen_logp[i] = 0 end @@ -107,9 +112,22 @@ function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf) end end end + + if p_conf.word_prob_report == true then + for i = 1, batch_size do + for t = 1, chunk_size do + if feeds.labels_s[t][i] ~= global_conf.vocab.null_token then + nerv.printf("LMTrainer.lm_process_file_rnn: word_prob_report_output, %.8f %s\n", word_prob[t][i], feeds.labels_s[t][i]) + end + end + end + end + if p_conf.one_sen_report == true then for i = 1, batch_size do - nerv.printf("LMTrainer.lm_process_file_rnn: one_sen_report_output, %f\n", sen_logp[i]) + if feeds.labels_s[1][i] ~= global_conf.vocab.null_token then + nerv.printf("LMTrainer.lm_process_file_rnn: one_sen_report_output, %f\n", sen_logp[i]) + end end end @@ -156,13 +174,16 @@ function LMTrainer.lm_process_file_birnn(global_conf, fn, tnn, do_train, p_conf) local reader local chunk_size, batch_size local r_conf = {["se_mode"] = true} + if p_conf.compressed_label ~= nil then + r_conf.compressed_label = p_conf.compressed_label + end if p_conf.one_sen_report == true then --report log prob one by one sentence if do_train == true then nerv.warning("LMTrainer.lm_process_file_birnn: warning, one_sen_report is true while do_train is also true, strange") end - nerv.printf("lm_process_file_birnn: one_sen report mode, set batch_size to 1 and chunk_size to max_sen_len(%d)\n", + nerv.printf("lm_process_file_birnn: one_sen report mode, set chunk_size to max_sen_len(%d)\n", global_conf.max_sen_len) - batch_size = 1 + batch_size = global_conf.batch_size chunk_size = global_conf.max_sen_len else batch_size = global_conf.batch_size @@ -196,7 +217,6 @@ function LMTrainer.lm_process_file_birnn(global_conf, fn, tnn, do_train, p_conf) if r == false then break end - for t = 1, chunk_size do tnn.err_inputs_m[t][1]:fill(1) for i = 1, batch_size do @@ -240,13 +260,17 @@ function LMTrainer.lm_process_file_birnn(global_conf, fn, tnn, do_train, p_conf) end if p_conf.one_sen_report == true then for i = 1, batch_size do - nerv.printf("LMTrainer.lm_process_file_birnn: one_sen_report_output, %f\n", sen_logp[i]) + if sen_logp[i] ~= nil then + nerv.printf("LMTrainer.lm_process_file_birnn: one_sen_report_output, %f\n", sen_logp[i]) + end end end --tnn:move_right_to_nextmb({0}) --do not need history for bi directional model global_conf.timer:toc('tnn_afterprocess') + --tnn:flush_all() --you need this for bilstmlm_ptb_v2, because it has connection across 2 time steps + global_conf.timer:toc('most_out_loop_lmprocessfile') --print log diff --git a/nerv/examples/lmptb/lmptb/layer/affine_recurrent.lua b/nerv/examples/lmptb/lmptb/layer/affine_recurrent.lua deleted file mode 100644 index 0a762f0..0000000 --- a/nerv/examples/lmptb/lmptb/layer/affine_recurrent.lua +++ /dev/null @@ -1,93 +0,0 @@ -local Recurrent = nerv.class('nerv.AffineRecurrentLayer', 'nerv.Layer') - ---id: string ---global_conf: table ---layer_conf: table ---Get Parameters -function Recurrent:__init(id, global_conf, layer_conf) - self.id = id - self.dim_in = layer_conf.dim_in - self.dim_out = layer_conf.dim_out - self.gconf = global_conf - - self.bp = layer_conf.bp - self.ltp_ih = layer_conf.ltp_ih --from input to hidden - self.ltp_hh = layer_conf.ltp_hh --from hidden to hidden - - self:check_dim_len(2, 1) - self.direct_update = layer_conf.direct_update -end - ---Check parameter -function Recurrent:init(batch_size) - if (self.ltp_ih.trans:ncol() ~= self.bp.trans:ncol() or - self.ltp_hh.trans:ncol() ~= self.bp.trans:ncol()) then - nerv.error("mismatching dimensions of ltp and bp") - end - if (self.dim_in[1] ~= self.ltp_ih.trans:nrow() or - self.dim_in[2] ~= self.ltp_hh.trans:nrow()) then - nerv.error("mismatching dimensions of ltp and input") - end - if (self.dim_out[1] ~= self.bp.trans:ncol()) then - nerv.error("mismatching dimensions of bp and output") - end - - self.ltp_ih_grad = self.ltp_ih.trans:create() - self.ltp_hh_grad = self.ltp_hh.trans:create() - self.ltp_ih:train_init() - self.ltp_hh:train_init() - self.bp:train_init() -end - -function Recurrent:update(bp_err, input, output) - if (self.direct_update == true) then - local ltp_ih = self.ltp_ih.trans - local ltp_hh = self.ltp_hh.trans - local bp = self.bp.trans - local ltc_ih = self.ltc_ih - local ltc_hh = self.ltc_hh - local bc = self.bc - local gconf = self.gconf - -- momentum gain - local mmt_gain = 1.0 / (1.0 - gconf.momentum); - local n = input[1]:nrow() * mmt_gain - -- update corrections (accumulated errors) - self.ltp_ih.correction:mul(input[1], bp_err[1], 1.0, gconf.momentum, 'T', 'N') - self.ltc_hh.correction:mul(input[2], bp_err[1], 1.0, gconf.momentum, 'T', 'N') - self.bp.correction:add(bc, bp_err[1]:colsum(), gconf.momentum, 1.0) - -- perform update - ltp_ih:add(ltp_ih, self.ltp_ih.correction, 1.0, -gconf.lrate / n) - ltp_hh:add(ltp_hh, self.ltp_hh.correction, 1.0, -gconf.lrate / n) - bp:add(bp, self.bp.correction, 1.0, -gconf.lrate / n) - -- weight decay - ltp_ih:add(ltp_ih, ltp_ih, 1.0, -gconf.lrate * gconf.wcost) - ltp_hh:add(ltp_hh, ltp_hh, 1.0, -gconf.lrate * gconf.wcost) - else - self.ltp_ih_grad:mul(input[1], bp_err[1], 1.0, 0.0, 'T', 'N') - self.ltp_ih:update(self.ltp_ih_grad) - self.ltp_hh_grad:mul(input[2], bp_err[1], 1.0, 0.0, 'T', 'N') - self.ltp_hh:update(self.ltp_hh_grad) - self.bp:update(bp_err[1]:colsum()) - end -end - -function Recurrent:propagate(input, output) - output[1]:mul(input[1], self.ltp_ih.trans, 1.0, 0.0, 'N', 'N') - output[1]:mul(input[2], self.ltp_hh.trans, 1.0, 1.0, 'N', 'N') - output[1]:add_row(self.bp.trans, 1.0) -end - -function Recurrent:back_propagate(bp_err, next_bp_err, input, output) - next_bp_err[1]:mul(bp_err[1], self.ltp_ih.trans, 1.0, 0.0, 'N', 'T') - next_bp_err[2]:mul(bp_err[1], self.ltp_hh.trans, 1.0, 0.0, 'N', 'T') - for i = 0, next_bp_err[2]:nrow() - 1 do - for j = 0, next_bp_err[2]:ncol() - 1 do - if (next_bp_err[2][i][j] > 10) then next_bp_err[2][i][j] = 10 end - if (next_bp_err[2][i][j] < -10) then next_bp_err[2][i][j] = -10 end - end - end -end - -function Recurrent:get_params() - return {self.ltp_ih, self.ltp_hh, self.bp} -end diff --git a/nerv/examples/lmptb/lmptb/layer/affine_recurrent_plusvec.lua b/nerv/examples/lmptb/lmptb/layer/affine_recurrent_plusvec.lua new file mode 100644 index 0000000..5606a09 --- /dev/null +++ b/nerv/examples/lmptb/lmptb/layer/affine_recurrent_plusvec.lua @@ -0,0 +1,74 @@ +local RecurrentV = nerv.class('nerv.AffineRecurrentPlusVecLayer', 'nerv.Layer') + +--id: string +--global_conf: table +--layer_conf: table +--Get Parameters +function RecurrentV:__init(id, global_conf, layer_conf) + self.id = id + self.dim_in = layer_conf.dim_in + self.dim_out = layer_conf.dim_out + self.gconf = global_conf + + self.bp = self:find_param("bp", layer_conf, global_conf, nerv.BiasParam, {1, self.dim_out[1]}) --layer_conf.bp + self.ltp_hh = self:find_param("ltp_hh", layer_conf, global_conf, nerv.LinearTransParam, {self.dim_in[2], self.dim_out[1]}) --layer_conf.ltp_hh --from hidden to hidden + + self:check_dim_len(2, 1) + self.direct_update = layer_conf.direct_update + + self.clip = layer_conf.clip --clip error in back_propagate +end + +--Check parameter +function RecurrentV:init(batch_size) + if (self.ltp_hh.trans:ncol() ~= self.bp.trans:ncol()) then + nerv.error("mismatching dimensions of ltp and bp") + end + if (self.dim_in[1] ~= self.ltp_hh.trans:nrow() or + self.dim_in[2] ~= self.ltp_hh.trans:nrow()) then + nerv.error("mismatching dimensions of ltp and input") + end + if (self.dim_out[1] ~= self.bp.trans:ncol()) then + nerv.error("mismatching dimensions of bp and output") + end + + self.ltp_hh_grad = self.ltp_hh.trans:create() + self.ltp_hh:train_init() + self.bp:train_init() +end + +function RecurrentV:batch_resize(batch_size) + -- do nothing +end + +function RecurrentV:update(bp_err, input, output) + --self.ltp_hh_grad:mul(input[2], bp_err[1], 1.0, 0.0, 'T', 'N') + self.ltp_hh:update_by_err_input(bp_err[1], input[2]) + self.bp:update_by_gradient(bp_err[1]:colsum()) +end + +function RecurrentV:propagate(input, output) + output[1]:copy_fromd(input[1]) + output[1]:mul(input[2], self.ltp_hh.trans, 1.0, 1.0, 'N', 'N') + output[1]:add_row(self.bp.trans, 1.0) +end + +function RecurrentV:back_propagate(bp_err, next_bp_err, input, output) + next_bp_err[1]:copy_fromd(bp_err[1]) + next_bp_err[2]:mul(bp_err[1], self.ltp_hh.trans, 1.0, 0.0, 'N', 'T') + --[[ + for i = 0, next_bp_err[2]:nrow() - 1 do + for j = 0, next_bp_err[2]:ncol() - 1 do + if (next_bp_err[2][i][j] > 10) then next_bp_err[2][i][j] = 10 end + if (next_bp_err[2][i][j] < -10) then next_bp_err[2][i][j] = -10 end + end + end + ]]-- + if (self.clip ~= nil) then + next_bp_err[2]:clip(- self.clip, self.clip) + end +end + +function RecurrentV:get_params() + return nerv.ParamRepo({self.ltp_hh, self.bp}) +end diff --git a/nerv/examples/lmptb/lmptb/layer/gru_t.lua b/nerv/examples/lmptb/lmptb/layer/gru_t.lua new file mode 100644 index 0000000..8f15cc8 --- /dev/null +++ b/nerv/examples/lmptb/lmptb/layer/gru_t.lua @@ -0,0 +1,114 @@ +local GRULayerT = nerv.class('nerv.GRULayerT', 'nerv.LayerT') + +function GRULayerT:__init(id, global_conf, layer_conf) + --input1:x input2:h input3:c(h^~) + self.id = id + self.dim_in = layer_conf.dim_in + self.dim_out = layer_conf.dim_out + self.gconf = global_conf + + if self.dim_in[2] ~= self.dim_out[1] then + nerv.error("dim_in[2](%d) mismatch with dim_out[1](%d)", self.dim_in[2], self.dim_out[1]) + end + + --prepare a DAGLayerT to hold the lstm structure + local pr = layer_conf.pr + if pr == nil then + pr = nerv.ParamRepo() + end + + local function ap(str) + return self.id .. '.' .. str + end + + local layers = { + ["nerv.CombinerLayer"] = { + [ap("inputXDup")] = {{}, {["dim_in"] = {self.dim_in[1]}, + ["dim_out"] = {self.dim_in[1], self.dim_in[1], self.dim_in[1]}, ["lambda"] = {1}}}, + [ap("inputHDup")] = {{}, {["dim_in"] = {self.dim_in[2]}, + ["dim_out"] = {self.dim_in[2], self.dim_in[2], self.dim_in[2], self.dim_in[2], self.dim_in[2]}, ["lambda"] = {1}}}, + [ap("updateGDup")] = {{}, {["dim_in"] = {self.dim_in[2]}, + ["dim_out"] = {self.dim_in[2], self.dim_in[2]}, ["lambda"] = {1}}}, + [ap("updateMergeL")] = {{}, {["dim_in"] = {self.dim_in[2], self.dim_in[2], self.dim_in[2]}, ["dim_out"] = {self.dim_out[1]}, + ["lambda"] = {1, -1, 1}}}, + }, + ["nerv.AffineLayer"] = { + [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, ["dim_out"] = {self.dim_out[1]}, ["pr"] = pr}}, + }, + ["nerv.TanhLayer"] = { + [ap("mainTanhL")] = {{}, {["dim_in"] = {self.dim_out[1]}, ["dim_out"] = {self.dim_out[1]}}}, + }, + ["nerv.GateFLayer"] = { + [ap("resetGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, + ["dim_out"] = {self.dim_in[2]}, ["pr"] = pr}}, + [ap("updateGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, + ["dim_out"] = {self.dim_in[2]}, ["pr"] = pr}}, + }, + ["nerv.ElemMulLayer"] = { + [ap("resetGMulL")] = {{}, {["dim_in"] = {self.dim_in[2], self.dim_in[2]}, ["dim_out"] = {self.dim_in[2]}}}, + [ap("updateGMulCL")] = {{}, {["dim_in"] = {self.dim_in[2], self.dim_in[2]}, ["dim_out"] = {self.dim_in[2]}}}, + [ap("updateGMulHL")] = {{}, {["dim_in"] = {self.dim_in[2], self.dim_in[2]}, ["dim_out"] = {self.dim_in[2]}}}, + }, + } + + local layerRepo = nerv.LayerRepo(layers, pr, global_conf) + + local connections_t = { + ["<input>[1]"] = ap("inputXDup[1]"), + ["<input>[2]"] = ap("inputHDup[1]"), + + [ap("inputXDup[1]")] = ap("resetGateL[1]"), + [ap("inputHDup[1]")] = ap("resetGateL[2]"), + [ap("inputXDup[2]")] = ap("updateGateL[1]"), + [ap("inputHDup[2]")] = ap("updateGateL[2]"), + [ap("updateGateL[1]")] = ap("updateGDup[1]"), + + [ap("resetGateL[1]")] = ap("resetGMulL[1]"), + [ap("inputHDup[3]")] = ap("resetGMulL[2]"), + + [ap("inputXDup[3]")] = ap("mainAffineL[1]"), + [ap("resetGMulL[1]")] = ap("mainAffineL[2]"), + [ap("mainAffineL[1]")] = ap("mainTanhL[1]"), + + [ap("updateGDup[1]")] = ap("updateGMulHL[1]"), + [ap("inputHDup[4]")] = ap("updateGMulHL[2]"), + [ap("updateGDup[2]")] = ap("updateGMulCL[1]"), + [ap("mainTanhL[1]")] = ap("updateGMulCL[2]"), + + [ap("inputHDup[5]")] = ap("updateMergeL[1]"), + [ap("updateGMulHL[1]")] = ap("updateMergeL[2]"), + [ap("updateGMulCL[1]")] = ap("updateMergeL[3]"), + + [ap("updateMergeL[1]")] = "<output>[1]", + } + + self.dagL = nerv.DAGLayerT(self.id, global_conf, + {["dim_in"] = self.dim_in, ["dim_out"] = self.dim_out, ["sub_layers"] = layerRepo, + ["connections"] = connections_t}) + + self:check_dim_len(2, 1) -- x, h and h +end + +function GRULayerT:init(batch_size, chunk_size) + self.dagL:init(batch_size, chunk_size) +end + +function GRULayerT:batch_resize(batch_size, chunk_size) + self.dagL:batch_resize(batch_size, chunk_size) +end + +function GRULayerT:update(bp_err, input, output, t) + self.dagL:update(bp_err, input, output, t) +end + +function GRULayerT:propagate(input, output, t) + self.dagL:propagate(input, output, t) +end + +function GRULayerT:back_propagate(bp_err, next_bp_err, input, output, t) + self.dagL:back_propagate(bp_err, next_bp_err, input, output, t) +end + +function GRULayerT:get_params() + return self.dagL:get_params() +end diff --git a/nerv/examples/lmptb/lmptb/layer/init.lua b/nerv/examples/lmptb/lmptb/layer/init.lua index ff29126..b345244 100644 --- a/nerv/examples/lmptb/lmptb/layer/init.lua +++ b/nerv/examples/lmptb/lmptb/layer/init.lua @@ -1,5 +1,6 @@ require 'lmptb.layer.select_linear' ---require 'lmptb.layer.affine_recurrent' +require 'lmptb.layer.affine_recurrent_plusvec' +require 'lmptb.layer.gru_t' require 'lmptb.layer.lm_affine_recurrent' diff --git a/nerv/examples/lmptb/lmptb/layer/select_linear.lua b/nerv/examples/lmptb/lmptb/layer/select_linear.lua index 580b9c5..f07eb2f 100644 --- a/nerv/examples/lmptb/lmptb/layer/select_linear.lua +++ b/nerv/examples/lmptb/lmptb/layer/select_linear.lua @@ -30,15 +30,15 @@ function SL:init(batch_size) end function SL:update(bp_err, input, output) - --use this to produce reproducable result + --use this to produce reproducable result, don't forget to set the dropout to zero! --for i = 1, input[1]:nrow(), 1 do -- local word_vec = self.ltp.trans[input[1][i - 1][0]] -- word_vec:add(word_vec, bp_err[1][i - 1], 1, - self.gconf.lrate / self.gconf.batch_size) --end --I tried the update_select_rows kernel which uses atomicAdd, but it generates unreproducable result - self.ltp.trans:update_select_rows(bp_err[1], input[1]:trans(), - self.gconf.lrate / self.gconf.batch_size, 0) - self.ltp.trans:add(self.ltp.trans, self.ltp.trans, 1.0, - self.gconf.lrate * self.gconf.wcost / self.gconf.batch_size) + self.ltp.trans:update_select_rows_by_colidx(bp_err[1], input[1], - self.gconf.lrate / self.gconf.batch_size, 0) + self.ltp.trans:add(self.ltp.trans, self.ltp.trans, 1.0, - self.gconf.lrate * self.gconf.wcost) end function SL:propagate(input, output) @@ -49,7 +49,7 @@ function SL:propagate(input, output) -- output[1][i]:fill(0) -- end --end - output[1]:copy_rows_fromd_by_idx(self.ltp.trans, input[1]:trans()) + output[1]:copy_rows_fromd_by_colidx(self.ltp.trans, input[1]) end function SL:back_propagate(bp_err, next_bp_err, input, output) diff --git a/nerv/examples/lmptb/lmptb/lmfeeder.lua b/nerv/examples/lmptb/lmptb/lmfeeder.lua index 34631bf..e140f38 100644 --- a/nerv/examples/lmptb/lmptb/lmfeeder.lua +++ b/nerv/examples/lmptb/lmptb/lmfeeder.lua @@ -1,4 +1,5 @@ require 'lmptb.lmvocab' +require 'lmptb.lmutil' local Feeder = nerv.class("nerv.LMFeeder") @@ -39,7 +40,7 @@ function Feeder:refresh_stream(id) local st = self.streams[id] if (st.store[st.head] ~= nil) then return end if (self.fh == nil) then return end - local list = self.vocab:read_line(self.fh) + local list = nerv.LMUtil.read_line(self.fh) if (list == nil) then --file has end printf("%s file expires, closing.\n", self.log_pre) self.fh:close() diff --git a/nerv/examples/lmptb/lmptb/lmseqreader.lua b/nerv/examples/lmptb/lmptb/lmseqreader.lua index ed791d2..0f29f8b 100644 --- a/nerv/examples/lmptb/lmptb/lmseqreader.lua +++ b/nerv/examples/lmptb/lmptb/lmseqreader.lua @@ -1,4 +1,5 @@ require 'lmptb.lmvocab' +require 'lmptb.lmutil' --require 'tnn.init' local LMReader = nerv.class("nerv.LMSeqReader") @@ -23,6 +24,10 @@ function LMReader:__init(global_conf, batch_size, chunk_size, vocab, r_conf) if r_conf.se_mode == true then self.se_mode = true end + self.compressed_label = false + if r_conf.compressed_label == true then + self.compressed_label = true + end end --fn: string @@ -45,6 +50,9 @@ function LMReader:open_file(fn) for j = 1, self.chunk_size, 1 do self.bak_inputs_m[j] = {} self.bak_inputs_m[j][1] = self.gconf.mmat_type(self.batch_size, 1) + if self.compressed_label == true then + self.bak_inputs_m[j][2] = self.gconf.mmat_type(self.batch_size, 1) + end --self.bak_inputs_m[j][2] = self.gconf.mmat_type(self.batch_size, self.vocab:size()) --since MMatrix does not yet have fill, this m[j][2] is not used end end @@ -58,7 +66,7 @@ function LMReader:refresh_stream(id) local st = self.streams[id] if (st.store[st.head] ~= nil) then return end if (self.fh == nil) then return end - local list = self.vocab:read_line(self.fh) + local list = nerv.LMUtil.read_line(self.fh) if (list == nil) then --file has end printf("%s file expires, closing.\n", self.log_pre) self.fh:close() @@ -117,6 +125,9 @@ function LMReader:get_batch(feeds) end inputs_s[j][i] = self.vocab.null_token self.bak_inputs_m[j][1][i - 1][0] = 0 + if self.compressed_label == true then + self.bak_inputs_m[j][2][i - 1][0] = 0 + end labels_s[j][i] = self.vocab.null_token else self:refresh_stream(i) @@ -131,7 +142,11 @@ function LMReader:get_batch(feeds) end if st.store[st.head + 1] ~= nil then labels_s[j][i] = st.store[st.head + 1] - inputs_m[j][2][i - 1][self.vocab:get_word_str(st.store[st.head + 1]).id - 1] = 1 + if self.compressed_label == true then + self.bak_inputs_m[j][2][i - 1][0] = self.vocab:get_word_str(st.store[st.head + 1]).id - 1 + else + inputs_m[j][2][i - 1][self.vocab:get_word_str(st.store[st.head + 1]).id - 1] = 1 + end else if (inputs_s[j][i] ~= self.vocab.null_token) then nerv.error("reader error : input not null but label is null_token") @@ -168,6 +183,9 @@ function LMReader:get_batch(feeds) flagsPack[j] = bit.bor(flagsPack[j], flags[j][i]) end inputs_m[j][1]:copy_fromh(self.bak_inputs_m[j][1]) + if self.compressed_label == true then + inputs_m[j][2]:copy_fromh(self.bak_inputs_m[j][2]) + end end --check for self.al_sen_start diff --git a/nerv/examples/lmptb/lmptb/lmutil.lua b/nerv/examples/lmptb/lmptb/lmutil.lua index 71e8e17..6d66d6e 100644 --- a/nerv/examples/lmptb/lmptb/lmutil.lua +++ b/nerv/examples/lmptb/lmptb/lmutil.lua @@ -1,11 +1,38 @@ local Util = nerv.class("nerv.LMUtil") +local mysplit = function(inputstr, sep) + if sep == nil then + sep = "%s" + end + local t={} ; i=1 + for str in string.gmatch(inputstr, "([^"..sep.."]+)") do + t[i] = str + i = i + 1 + end + return t +end + --function rounds a number to the given number of decimal places. function Util.round(num, idp) local mult = 10^(idp or 0) return math.floor(num * mult + 0.5) / mult end +--fh: file_handle +--Returns: a list of tokens(string) in the line, if there is no "</s>" at the end, the function will at it, if nothing to read, returns nil +function Util.read_line(fh) + local l_str, list + + repeat + l_str = fh:read("*line") + if (l_str == nil) then return nil end + list = mysplit(l_str) + until #list >= 1 + + return list +end + + --list: table, list of string(word) --vocab: nerv.LMVocab --ty: nerv.CuMatrix @@ -114,7 +141,7 @@ function Result:logp_sample(cla) end function Result:status(cla) - return "LMResult status of " .. cla .. ": " .. "<SEN_CN " .. self[cla].cn_sen .. "> <W_CN " .. self[cla].cn_w .. "> <PPL_NET " .. self:ppl_net(cla) .. "> <PPL_OOV " .. self:ppl_all(cla) .. "> <LOGP " .. self[cla].logp_all .. ">" + return "LMResult status of " .. cla .. ": " .. "<SEN_CN " .. self[cla].cn_sen .. "> <W_CN " .. self[cla].cn_w .. "> <UNK_CN " .. self[cla].cn_unk .. "> <PPL_NET " .. self:ppl_net(cla) .. "> <PPL_OOV " .. self:ppl_all(cla) .. "> <LOGP " .. self[cla].logp_all .. ">" end local Timer = nerv.class("nerv.Timer") diff --git a/nerv/examples/lmptb/lmptb/lmvocab.lua b/nerv/examples/lmptb/lmptb/lmvocab.lua index 3d256c0..0e7ef3e 100644 --- a/nerv/examples/lmptb/lmptb/lmvocab.lua +++ b/nerv/examples/lmptb/lmptb/lmvocab.lua @@ -1,3 +1,5 @@ +require 'lmptb.lmutil' + local Vocab = nerv.class("nerv.LMVocab") local printf = nerv.printf @@ -98,19 +100,7 @@ function Vocab:get_word_id(key) if (self.map_id[key] == nil) then nerv.error("id key %d does not exist.", key) end - return self.map_id(key) -end - ---fh: file_handle ---Returns: a list of tokens(string) in the line, if there is no "</s>" at the end, the function will at it, if nothing to read, returns nil -function Vocab:read_line(fh) - local l_str = fh:read("*line") - if (l_str == nil) then return nil end - local list = mysplit(l_str) - if (list[(#list)] ~= self.sen_end_token) then - list[#list + 1] = self.sen_end_token - end - return list + return self.map_id[key] end --fn: string @@ -119,7 +109,7 @@ function Vocab:build_file(fn) printf("%s Vocab building on file %s...\n", self.log_pre, fn) local file = io.open(fn, "r") while (true) do - local list = self:read_line(file) + local list = nerv.LMUtil.read_line(file) if (list == nil) then break else diff --git a/nerv/examples/lmptb/lmptb/lstm_t_v2.lua b/nerv/examples/lmptb/lmptb/lstm_t_v2.lua index dc2fe45..e7bf3a7 100644 --- a/nerv/examples/lmptb/lmptb/lstm_t_v2.lua +++ b/nerv/examples/lmptb/lmptb/lstm_t_v2.lua @@ -1,5 +1,4 @@ -local LSTMLayerT = nerv.class('nerv.LSTMLayerTv2', 'nerv.LayerT') ---a version of LSTM that only feed h into the gates +local LSTMLayerT = nerv.class('nerv.LSTMLayerV2T', 'nerv.LayerT') function LSTMLayerT:__init(id, global_conf, layer_conf) --input1:x input2:h input3:c @@ -21,13 +20,13 @@ function LSTMLayerT:__init(id, global_conf, layer_conf) local layers = { ["nerv.CombinerLayer"] = { [ap("inputXDup")] = {{}, {["dim_in"] = {self.dim_in[1]}, - ["dim_out"] = {self.dim_in[1], self.dim_in[1], self.dim_in[1], self.dim_in[1]}, ["lambda"] = {1}}}, + ["dim_out"] = {self.dim_in[1], self.dim_in[1], self.dim_in[1]}, ["lambda"] = {1}}}, [ap("inputHDup")] = {{}, {["dim_in"] = {self.dim_in[2]}, - ["dim_out"] = {self.dim_in[2], self.dim_in[2], self.dim_in[2], self.dim_in[2]}, ["lambda"] = {1}}}, - [ap("inputCDup")] = {{}, {["dim_in"] = {self.dim_in[3]}, - ["dim_out"] = {self.dim_in[3]}, ["lambda"] = {1}}}, - [ap("mainCDup")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, - ["dim_out"] = {self.dim_in[3], self.dim_in[3]}, ["lambda"] = {1, 1}}}, + ["dim_out"] = {self.dim_in[2], self.dim_in[2], self.dim_in[2]}, ["lambda"] = {1}}}, + --[ap("inputCDup")] = {{}, {["dim_in"] = {self.dim_in[3]}, + --["dim_out"] = {self.dim_in[3], self.dim_in[3], self.dim_in[3]}, ["lambda"] = {1}}}, + [ap("mainCDup")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3], self.dim_in[3]}, + ["lambda"] = {1, 1}}}, }, ["nerv.AffineLayer"] = { [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, @@ -42,14 +41,14 @@ function LSTMLayerT:__init(id, global_conf, layer_conf) ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}}, [ap("inputGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}}, - [ap("outputGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, - ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}}, + --[ap("outputGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2], self.dim_in[3]}, + -- ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}}, }, ["nerv.ElemMulLayer"] = { [ap("inputGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}}, [ap("forgetGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}}, - [ap("outputGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}}, + --[ap("outputGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}}, }, } @@ -58,38 +57,36 @@ function LSTMLayerT:__init(id, global_conf, layer_conf) local connections_t = { ["<input>[1]"] = ap("inputXDup[1]"), ["<input>[2]"] = ap("inputHDup[1]"), - ["<input>[3]"] = ap("inputCDup[1]"), [ap("inputXDup[1]")] = ap("mainAffineL[1]"), [ap("inputHDup[1]")] = ap("mainAffineL[2]"), - [ap("mainAffineL[1]")] = ap("mainTanhL[1]"), [ap("inputXDup[2]")] = ap("inputGateL[1]"), [ap("inputHDup[2]")] = ap("inputGateL[2]"), + --[ap("inputCDup[1]")] = ap("inputGateL[3]"), [ap("inputXDup[3]")] = ap("forgetGateL[1]"), [ap("inputHDup[3]")] = ap("forgetGateL[2]"), + --[ap("inputCDup[2]")] = ap("forgetGateL[3]"), [ap("mainTanhL[1]")] = ap("inputGMulL[1]"), [ap("inputGateL[1]")] = ap("inputGMulL[2]"), - [ap("inputCDup[1]")] = ap("forgetGMulL[1]"), + [ap("<input>[3]")] = ap("forgetGMulL[1]"), [ap("forgetGateL[1]")] = ap("forgetGMulL[2]"), [ap("inputGMulL[1]")] = ap("mainCDup[1]"), [ap("forgetGMulL[1]")] = ap("mainCDup[2]"), - [ap("inputXDup[4]")] = ap("outputGateL[1]"), - [ap("inputHDup[4]")] = ap("outputGateL[2]"), + --[ap("inputXDup[4]")] = ap("outputGateL[1]"), + --[ap("inputHDup[4]")] = ap("outputGateL[2]"), + --[ap("mainCDup[3]")] = ap("outputGateL[3]"), [ap("mainCDup[2]")] = "<output>[2]", [ap("mainCDup[1]")] = ap("outputTanhL[1]"), - [ap("outputTanhL[1]")] = ap("outputGMulL[1]"), - [ap("outputGateL[1]")] = ap("outputGMulL[2]"), - - [ap("outputGMulL[1]")] = "<output>[1]", + [ap("outputTanhL[1]")] = "<output>[1]", } self.dagL = nerv.DAGLayerT(self.id, global_conf, {["dim_in"] = self.dim_in, ["dim_out"] = self.dim_out, ["sub_layers"] = layerRepo, diff --git a/nerv/examples/lmptb/logs/LOG-tnn-h300 b/nerv/examples/lmptb/logs/LOG-tnn-h300 index 77fd612..3e99c5d 100644 --- a/nerv/examples/lmptb/logs/LOG-tnn-h300 +++ b/nerv/examples/lmptb/logs/LOG-tnn-h300 @@ -1,69 +1,64 @@ Greetings -[SCHEDULER]: applying arg[2](global_conf.hidden_size=300)... +[SCHEDULER]: not user setting, all default... [SCHEDULER]: printing global_conf... +log_fn /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/log_lstm_tnn_train-test_TT12_23_15-25-01 test_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds train_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.train.txt.adds -param_random function: 0x40237598 -train_fn_shuf_bak /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf_bak -decay_iter 16 -mmat_type table: 0x409280f8 +param_random function: 0x41a72790 +train_fn_shuf_bak /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf_bak +decay_iter 15 +mmat_type table: 0x4148c690 vocab_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/vocab -train_fn_shuf /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf -param_fn /home/slhome/txh18/workspace/nerv/play/dagL_test/params +train_fn_shuf /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf +work_dir_base /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnn +param_fn /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params log_w_num 40000 -work_dir /home/slhome/txh18/workspace/nerv/play/dagL_test +work_dir /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06 batch_size 10 hidden_size 300 timer nerv.Timer sche_log_pre [SCHEDULER]: nn_act_default 0 -max_iter 35 +max_iter 30 valid_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds lrate 1 momentum 0 -wcost 1e-05 +wcost 1e-06 chunk_size 15 -cumat_type table: 0x40935010 +cumat_type table: 0x4149e6a8 [SCHEDULER]: printing training scheduling options... lr_half false start_iter -1 +test_iter -1 ppl_last 100000 [SCHEDULER]: printing training scheduling end. [SCHEDULER]: creating work_dir... +(15:25:07 2015-12-23)[nerv] info: CAUTION[LOG_REDIRECT], all nerv.printf/info/warning/error calls will be double-written to /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/log_lstm_tnn_train-test_TT12_23_15-25-01 [SCHEDULER]: building vocab... [LOG]LMVocab: Vocab building on file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/vocab... [LOG]LMVocab: Building finished, vocab size now is 10000. [SCHEDULER]: preparing parameters... -[SCHEDULER]: first time, generating parameters... +[SCHEDULER]: first time, prepare some pre-set parameters, and leaving other parameters to auto-generation... ===INITIAL VALIDATION=== [SCHEDULER]: preparing parameters... -[SCHEDULER]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.0... +[SCHEDULER]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.0... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 1 from 34510155 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 2 from 35545278 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 70045626 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 4 from 70049129 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 5 from 70164107 [SCHEDULER]: preparing parameters end. [SCHEDULER]: preparing layers... -(10:08:46 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:08:46 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:08:46 2015-11-18)[nerv] info: create layer: combinerL1 -(10:08:46 2015-11-18)[nerv] info: create layer: outputL -(10:08:46 2015-11-18)[nerv] info: create layer: softmaxL -(10:08:46 2015-11-18)[nerv] info: create layer: selectL1 +(15:25:10 2015-12-23)[nerv] info: create layer: sigmoidL1 +(15:25:10 2015-12-23)[nerv] info: create layer: combinerL1 +(15:25:10 2015-12-23)[nerv] info: create layer: selectL1 +(15:25:10 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] is not found in layer_conf or layer_conf.paramRepo, switch to auto-generate. +(15:25:22 2015-12-23)[nerv] info: create layer: outputL +(15:25:22 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] is not found in layer_conf or layer_conf.paramRepo, switch to auto-generate. +(15:25:35 2015-12-23)[nerv] info: Param [bp] of layer [outputL] is not found in layer_conf or layer_conf.paramRepo, switch to auto-generate. +(15:25:35 2015-12-23)[nerv] info: create layer: softmaxL +(15:25:35 2015-12-23)[nerv] info: create layer: recurrentL1 +(15:25:35 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] is not found in layer_conf or layer_conf.paramRepo, switch to auto-generate. +(15:25:35 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] is not found in layer_conf or layer_conf.paramRepo, switch to auto-generate. [SCHEDULER]: preparing layers end. [SCHEDULER]: Generate and initing TNN ... +(15:25:35 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -79,54 +74,70 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(15:25:35 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:25:35 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:25:35 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:25:35 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:25:35 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:25:35 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER]: Initing TNN end. [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER]: 40095 words processed Wed Nov 18 10:08:51 2015. - [SCHEDULER]: log prob per sample :-4.003230. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47666 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:4.10684 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.41463 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12208 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:25:37 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:25:37 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:25:37 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:25:37 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:25:37 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:25:37 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:25:37 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER]: 40095 words processed Wed Dec 23 15:25:45 2015. + [SCHEDULER]: log prob per sample :-3.991694. + [global_conf.timer]: time spent on tnn_beforeprocess:0.52286 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:7.15208 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.36437 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15894 clock time [LOG]LMSeqReader: file expires, closing. +(15:25:52 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER]: Displaying result: -[SCHEDULER]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 10545.667125173> <PPL_OOV 10092.059293004> <LOGP -295333.54956309> +[SCHEDULER]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 10204.583785681> <PPL_OOV 9812.8219182096> <LOGP -294434.71934724> [SCHEDULER]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER1 LR1]: preparing parameters... -[SCHEDULER ITER1 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.0... +[SCHEDULER ITER1 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.0... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34510155 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 1 from 3512 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 35545278 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 118553 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 70045626 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 3 from 1154327 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 70049129 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 4 from 35653985 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70164107 +reading chunk 5 from 70164146 [SCHEDULER ITER1 LR1]: preparing parameters end. [SCHEDULER ITER1 LR1]: preparing layers... -(10:08:58 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:08:58 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:08:58 2015-11-18)[nerv] info: create layer: combinerL1 -(10:08:58 2015-11-18)[nerv] info: create layer: outputL -(10:08:58 2015-11-18)[nerv] info: create layer: softmaxL -(10:08:58 2015-11-18)[nerv] info: create layer: selectL1 +(15:25:55 2015-12-23)[nerv] info: create layer: sigmoidL1 +(15:25:55 2015-12-23)[nerv] info: create layer: combinerL1 +(15:25:55 2015-12-23)[nerv] info: create layer: selectL1 +(15:25:55 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(15:25:55 2015-12-23)[nerv] info: create layer: outputL +(15:25:55 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(15:25:55 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(15:25:55 2015-12-23)[nerv] info: create layer: softmaxL +(15:25:55 2015-12-23)[nerv] info: create layer: recurrentL1 +(15:25:55 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(15:25:55 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER1 LR1]: preparing layers end. [SCHEDULER ITER1 LR1]: Generate and initing TNN ... +(15:25:55 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -142,253 +153,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(15:25:55 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:25:55 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:25:55 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:25:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:25:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:25:55 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER1 LR1]: Initing TNN end. ===ITERATION 1 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER1 LR1]: 40099 words processed Wed Nov 18 10:09:09 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-3.210428. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47135 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75538 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22335 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57185 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10319 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12114 clock time -[SCHEDULER ITER1 LR1]: 80063 words processed Wed Nov 18 10:09:21 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-3.048713. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46917 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75080 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.19680 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56019 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09740 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12030 clock time -[SCHEDULER ITER1 LR1]: 120068 words processed Wed Nov 18 10:09:33 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.984911. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47044 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74848 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.18711 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55472 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09624 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12070 clock time -[SCHEDULER ITER1 LR1]: 160017 words processed Wed Nov 18 10:09:45 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.930297. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47130 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75352 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20206 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55933 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09779 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12088 clock time -[SCHEDULER ITER1 LR1]: 200138 words processed Wed Nov 18 10:09:57 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.897628. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46695 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76538 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23399 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57699 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10442 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12134 clock time -[SCHEDULER ITER1 LR1]: 240007 words processed Wed Nov 18 10:10:09 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.867407. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46510 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73963 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.15285 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54229 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.08908 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12042 clock time -[SCHEDULER ITER1 LR1]: 280135 words processed Wed Nov 18 10:10:21 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.842329. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46714 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75314 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21984 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57472 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10567 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12149 clock time -[SCHEDULER ITER1 LR1]: 320080 words processed Wed Nov 18 10:10:33 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.823036. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46701 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75169 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20551 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56419 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10051 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12255 clock time -[SCHEDULER ITER1 LR1]: 360059 words processed Wed Nov 18 10:10:45 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.800986. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47814 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.79713 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.30428 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58719 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10865 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12425 clock time -[SCHEDULER ITER1 LR1]: 400021 words processed Wed Nov 18 10:10:57 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.786891. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45977 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73850 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.17294 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55793 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09813 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12008 clock time -[SCHEDULER ITER1 LR1]: 440102 words processed Wed Nov 18 10:11:09 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.770894. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46478 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75843 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22462 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57475 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10523 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12179 clock time -[SCHEDULER ITER1 LR1]: 480051 words processed Wed Nov 18 10:11:21 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.754795. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46079 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74221 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.17129 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55381 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09688 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12041 clock time -[SCHEDULER ITER1 LR1]: 520093 words processed Wed Nov 18 10:11:33 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.740832. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46013 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74097 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.19854 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57069 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10617 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12053 clock time -[SCHEDULER ITER1 LR1]: 560039 words processed Wed Nov 18 10:11:45 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.725416. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45905 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73400 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.16478 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55572 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09740 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12013 clock time -[SCHEDULER ITER1 LR1]: 600112 words processed Wed Nov 18 10:11:57 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.716177. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46687 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76335 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25973 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11483 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12277 clock time -[SCHEDULER ITER1 LR1]: 640076 words processed Wed Nov 18 10:12:09 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.705623. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46173 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74375 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.17842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55617 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09811 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12090 clock time -[SCHEDULER ITER1 LR1]: 680026 words processed Wed Nov 18 10:12:21 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.696685. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46080 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73873 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.16648 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55295 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09625 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12004 clock time -[SCHEDULER ITER1 LR1]: 720133 words processed Wed Nov 18 10:12:33 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.687998. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46041 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73861 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.19165 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57016 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10446 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12015 clock time -[SCHEDULER ITER1 LR1]: 760048 words processed Wed Nov 18 10:12:45 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.676801. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45850 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73148 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.15910 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55561 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09577 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11943 clock time -[SCHEDULER ITER1 LR1]: 800117 words processed Wed Nov 18 10:12:57 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.669544. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45975 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74013 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.19139 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56894 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10351 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12031 clock time -[SCHEDULER ITER1 LR1]: 840116 words processed Wed Nov 18 10:13:09 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.659602. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46031 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73769 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.17842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56008 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10197 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12018 clock time -[SCHEDULER ITER1 LR1]: 880037 words processed Wed Nov 18 10:13:20 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.652101. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45781 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73064 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.15936 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55415 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09854 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11997 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:25:55 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:25:55 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:25:55 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:25:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:25:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:25:55 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:25:55 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER1 LR1]: 40099 words processed Wed Dec 23 15:26:11 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.208733. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49912 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.18861 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.10438 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.86339 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13806 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15891 clock time +[SCHEDULER ITER1 LR1]: 80063 words processed Wed Dec 23 15:26:27 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.027452. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50375 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.16717 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.04599 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.83912 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12530 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15900 clock time +[SCHEDULER ITER1 LR1]: 120068 words processed Wed Dec 23 15:26:43 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.949047. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17205 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.04428 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.83331 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12965 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15888 clock time +[SCHEDULER ITER1 LR1]: 160017 words processed Wed Dec 23 15:27:00 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.892670. + [global_conf.timer]: time spent on tnn_beforeprocess:0.52759 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.48079 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.64937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:6.00684 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.16768 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16836 clock time +[SCHEDULER ITER1 LR1]: 200138 words processed Wed Dec 23 15:27:17 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.862344. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50400 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.26080 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.23420 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.90703 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13851 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16026 clock time +[SCHEDULER ITER1 LR1]: 240007 words processed Wed Dec 23 15:27:34 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.832988. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50224 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.27449 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.24373 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.89637 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13873 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15986 clock time +[SCHEDULER ITER1 LR1]: 280135 words processed Wed Dec 23 15:27:51 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.807892. + [global_conf.timer]: time spent on tnn_beforeprocess:0.52736 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.50277 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.71085 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:6.03449 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.17865 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16944 clock time +[SCHEDULER ITER1 LR1]: 320080 words processed Wed Dec 23 15:28:07 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.787610. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47566 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.02338 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73258 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75188 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10406 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14869 clock time +[SCHEDULER ITER1 LR1]: 360059 words processed Wed Dec 23 15:28:23 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.765522. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46910 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98256 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.72921 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09774 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14636 clock time +[SCHEDULER ITER1 LR1]: 400021 words processed Wed Dec 23 15:28:39 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.751868. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46969 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98096 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64490 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.72892 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09642 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14590 clock time +[SCHEDULER ITER1 LR1]: 440102 words processed Wed Dec 23 15:28:55 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.736409. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47132 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00369 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69775 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74976 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10222 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14619 clock time +[SCHEDULER ITER1 LR1]: 480051 words processed Wed Dec 23 15:29:11 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.721181. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46899 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98351 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.72954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09617 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14592 clock time +[SCHEDULER ITER1 LR1]: 520093 words processed Wed Dec 23 15:29:27 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.708133. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47288 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.01261 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72172 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75420 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10711 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14766 clock time +[SCHEDULER ITER1 LR1]: 560039 words processed Wed Dec 23 15:29:43 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.693347. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47246 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00098 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68805 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74210 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09885 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14709 clock time +[SCHEDULER ITER1 LR1]: 600112 words processed Wed Dec 23 15:29:59 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.684825. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47372 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.03642 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76671 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10646 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14797 clock time +[SCHEDULER ITER1 LR1]: 640076 words processed Wed Dec 23 15:30:16 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.674873. + [global_conf.timer]: time spent on tnn_beforeprocess:0.51025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.37618 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.42601 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.94153 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15211 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16617 clock time +[SCHEDULER ITER1 LR1]: 680026 words processed Wed Dec 23 15:30:33 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.666825. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50876 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.36958 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.46533 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.96944 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15677 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16815 clock time +[SCHEDULER ITER1 LR1]: 720133 words processed Wed Dec 23 15:30:50 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.658816. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49595 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.25148 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.18774 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.88182 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13916 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15690 clock time +[SCHEDULER ITER1 LR1]: 760048 words processed Wed Dec 23 15:31:07 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.648532. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50178 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.32493 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.32939 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.92035 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14429 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16224 clock time +[SCHEDULER ITER1 LR1]: 800117 words processed Wed Dec 23 15:31:24 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.642086. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50275 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.30792 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.31499 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.91857 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15170 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16169 clock time +[SCHEDULER ITER1 LR1]: 840116 words processed Wed Dec 23 15:31:41 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.633060. + [global_conf.timer]: time spent on tnn_beforeprocess:0.51405 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.40662 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.48276 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.95914 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15308 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16813 clock time +[SCHEDULER ITER1 LR1]: 880037 words processed Wed Dec 23 15:31:58 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.626273. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.27819 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.22222 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.87408 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14477 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15965 clock time [LOG]LMSeqReader: file expires, closing. +(15:32:02 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER1 LR1]: Displaying result: -[SCHEDULER ITER1 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 512.01508977938> <PPL_OOV 447.19970345812> <LOGP -2463877.0452922> -[SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER1 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 483.24236261194> <PPL_OOV 421.50130390255> <LOGP -2439984.2055128> +[SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER1 LR1]: shuffling training file ===PEEK ON TEST 1=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER1 LR1]: 40087 words processed Wed Nov 18 10:13:30 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.504076. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75798 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.45029 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11464 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:32:05 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:32:05 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:32:05 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:32:05 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:32:05 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:32:05 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:32:05 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER1 LR1]: 40087 words processed Wed Dec 23 15:32:13 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.490319. + [global_conf.timer]: time spent on tnn_beforeprocess:0.52150 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.47313 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:7.29086 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15853 clock time [LOG]LMSeqReader: file expires, closing. +(15:32:21 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER1 LR1]: Displaying result: -[SCHEDULER ITER1 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 370.53246252795> <PPL_OOV 323.54357826836> <LOGP -206893.75941108> +[SCHEDULER ITER1 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 359.03018078479> <PPL_OOV 313.38700895499> <LOGP -205751.95595632> [SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 1=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER1 LR1]: 40095 words processed Wed Nov 18 10:13:39 2015. - [SCHEDULER ITER1 LR1]: log prob per sample :-2.547314. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46703 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75630 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.44636 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11481 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:32:21 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:32:21 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:32:21 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:32:21 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:32:21 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:32:21 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:32:22 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER1 LR1]: 40095 words processed Wed Dec 23 15:32:30 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.537461. + [global_conf.timer]: time spent on tnn_beforeprocess:0.52284 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.48628 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:7.30197 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15646 clock time [LOG]LMSeqReader: file expires, closing. +(15:32:37 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER1 LR1]: Displaying result: -[SCHEDULER ITER1 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 369.92944180373> <PPL_OOV 331.71274183319> <LOGP -185931.4164841> +[SCHEDULER ITER1 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 361.36204459889> <PPL_OOV 324.02010167808> <LOGP -185179.78732431> [SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER1 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.1... +[SCHEDULER ITER1 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.1... [SCHEDULER ITER2 LR1]: preparing parameters... -[SCHEDULER ITER2 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.1... +[SCHEDULER ITER2 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.1... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1076633 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3667 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1080289 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120000 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35608824 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1196590 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35725140 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35732588 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70244205 +reading chunk 5 from 70253773 [SCHEDULER ITER2 LR1]: preparing parameters end. [SCHEDULER ITER2 LR1]: preparing layers... -(10:13:48 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:13:48 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:13:48 2015-11-18)[nerv] info: create layer: combinerL1 -(10:13:48 2015-11-18)[nerv] info: create layer: outputL -(10:13:48 2015-11-18)[nerv] info: create layer: softmaxL -(10:13:48 2015-11-18)[nerv] info: create layer: selectL1 +(15:32:42 2015-12-23)[nerv] info: create layer: sigmoidL1 +(15:32:42 2015-12-23)[nerv] info: create layer: combinerL1 +(15:32:42 2015-12-23)[nerv] info: create layer: selectL1 +(15:32:42 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(15:32:42 2015-12-23)[nerv] info: create layer: outputL +(15:32:42 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(15:32:42 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(15:32:42 2015-12-23)[nerv] info: create layer: softmaxL +(15:32:42 2015-12-23)[nerv] info: create layer: recurrentL1 +(15:32:42 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(15:32:42 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER2 LR1]: preparing layers end. [SCHEDULER ITER2 LR1]: Generate and initing TNN ... +(15:32:42 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -404,253 +451,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(15:32:42 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:32:42 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:32:42 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:32:42 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:32:42 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:32:42 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER2 LR1]: Initing TNN end. ===ITERATION 2 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER2 LR1]: 40092 words processed Wed Nov 18 10:13:59 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.541969. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46836 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72598 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22414 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56508 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09968 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11795 clock time -[SCHEDULER ITER2 LR1]: 80099 words processed Wed Nov 18 10:14:11 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.535649. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46987 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73527 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21641 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55211 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09413 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11941 clock time -[SCHEDULER ITER2 LR1]: 120004 words processed Wed Nov 18 10:14:23 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.527511. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46606 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72308 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.19836 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54952 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09336 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11779 clock time -[SCHEDULER ITER2 LR1]: 160114 words processed Wed Nov 18 10:14:35 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.522344. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46532 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72510 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22327 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56668 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10154 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11830 clock time -[SCHEDULER ITER2 LR1]: 200066 words processed Wed Nov 18 10:14:46 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.517881. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46174 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.71860 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.18739 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55076 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09280 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11754 clock time -[SCHEDULER ITER2 LR1]: 240045 words processed Wed Nov 18 10:14:58 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.512076. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46366 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72754 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55625 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09893 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11910 clock time -[SCHEDULER ITER2 LR1]: 280057 words processed Wed Nov 18 10:15:10 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.507676. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47597 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.79397 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.37668 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60190 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11988 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12311 clock time -[SCHEDULER ITER2 LR1]: 320106 words processed Wed Nov 18 10:15:22 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.500944. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73117 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22486 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56401 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10012 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11863 clock time -[SCHEDULER ITER2 LR1]: 360024 words processed Wed Nov 18 10:15:34 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.497026. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45968 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73210 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20736 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55524 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09260 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11888 clock time -[SCHEDULER ITER2 LR1]: 400089 words processed Wed Nov 18 10:15:46 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.493683. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74623 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24888 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56961 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10136 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11942 clock time -[SCHEDULER ITER2 LR1]: 440067 words processed Wed Nov 18 10:15:58 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.490451. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46117 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73519 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21602 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55836 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09430 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11994 clock time -[SCHEDULER ITER2 LR1]: 480051 words processed Wed Nov 18 10:16:10 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.487587. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45998 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73365 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20277 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09232 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11882 clock time -[SCHEDULER ITER2 LR1]: 520140 words processed Wed Nov 18 10:16:22 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.484627. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45975 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72819 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21976 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56564 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10028 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11841 clock time -[SCHEDULER ITER2 LR1]: 560132 words processed Wed Nov 18 10:16:33 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.481780. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45692 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.71231 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.16816 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54569 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09361 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11724 clock time -[SCHEDULER ITER2 LR1]: 600118 words processed Wed Nov 18 10:16:45 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.479088. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45915 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73072 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.18993 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54640 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09120 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11835 clock time -[SCHEDULER ITER2 LR1]: 640090 words processed Wed Nov 18 10:16:57 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.475904. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45833 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72455 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.18397 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54664 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09177 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11810 clock time -[SCHEDULER ITER2 LR1]: 680075 words processed Wed Nov 18 10:17:09 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.474017. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45720 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.71394 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.16784 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54518 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09097 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11778 clock time -[SCHEDULER ITER2 LR1]: 720043 words processed Wed Nov 18 10:17:21 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.471783. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45892 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72588 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.19146 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55005 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09232 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11806 clock time -[SCHEDULER ITER2 LR1]: 760012 words processed Wed Nov 18 10:17:33 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.469729. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73453 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21522 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55667 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09724 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11897 clock time -[SCHEDULER ITER2 LR1]: 800113 words processed Wed Nov 18 10:17:45 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.467617. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46246 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74674 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25543 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57267 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10430 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11946 clock time -[SCHEDULER ITER2 LR1]: 840089 words processed Wed Nov 18 10:17:57 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.465015. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46102 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74097 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21331 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55202 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09360 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11916 clock time -[SCHEDULER ITER2 LR1]: 880052 words processed Wed Nov 18 10:18:09 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.463152. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46119 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74489 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23141 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55830 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09725 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12007 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:32:42 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:32:42 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:32:42 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:32:42 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:32:42 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:32:42 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:32:42 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER2 LR1]: 40092 words processed Wed Dec 23 15:32:59 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.543962. + [global_conf.timer]: time spent on tnn_beforeprocess:0.51651 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.33365 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.44072 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.95534 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15673 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16276 clock time +[SCHEDULER ITER2 LR1]: 80099 words processed Wed Dec 23 15:33:16 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.529145. + [global_conf.timer]: time spent on tnn_beforeprocess:0.52334 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.38595 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.51567 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.96633 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15538 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16427 clock time +[SCHEDULER ITER2 LR1]: 120004 words processed Wed Dec 23 15:33:33 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.516662. + [global_conf.timer]: time spent on tnn_beforeprocess:0.51856 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.34632 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.46008 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.95776 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15115 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16214 clock time +[SCHEDULER ITER2 LR1]: 160114 words processed Wed Dec 23 15:33:50 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.511438. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.25231 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.27334 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.90898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14335 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15935 clock time +[SCHEDULER ITER2 LR1]: 200066 words processed Wed Dec 23 15:34:07 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.507731. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50431 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.26931 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.29652 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.91403 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13902 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16009 clock time +[SCHEDULER ITER2 LR1]: 240045 words processed Wed Dec 23 15:34:24 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.501616. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50609 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.27073 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.30914 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.92153 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13842 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16158 clock time +[SCHEDULER ITER2 LR1]: 280057 words processed Wed Dec 23 15:34:41 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.496402. + [global_conf.timer]: time spent on tnn_beforeprocess:0.51138 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.34649 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.46296 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.96916 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15271 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16003 clock time +[SCHEDULER ITER2 LR1]: 320106 words processed Wed Dec 23 15:34:58 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.489678. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50909 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.30965 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.40222 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.95191 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15167 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16134 clock time +[SCHEDULER ITER2 LR1]: 360024 words processed Wed Dec 23 15:35:15 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.485612. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50705 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.31150 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.39945 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.94661 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14825 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16326 clock time +[SCHEDULER ITER2 LR1]: 400089 words processed Wed Dec 23 15:35:32 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.482872. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50089 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.28666 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.34692 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.94173 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14700 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15709 clock time +[SCHEDULER ITER2 LR1]: 440067 words processed Wed Dec 23 15:35:49 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.480066. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50701 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.35039 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.43599 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.95120 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14437 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16307 clock time +[SCHEDULER ITER2 LR1]: 480051 words processed Wed Dec 23 15:36:06 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.477442. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49426 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.23795 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.23275 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.90809 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13161 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15627 clock time +[SCHEDULER ITER2 LR1]: 520140 words processed Wed Dec 23 15:36:23 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.474724. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50491 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.28851 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.36912 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.94956 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15115 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15991 clock time +[SCHEDULER ITER2 LR1]: 560132 words processed Wed Dec 23 15:36:40 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.472104. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49716 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.27276 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.28731 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.90787 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14236 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15943 clock time +[SCHEDULER ITER2 LR1]: 600118 words processed Wed Dec 23 15:36:57 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.470994. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50995 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.31916 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.39051 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.94040 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14252 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16319 clock time +[SCHEDULER ITER2 LR1]: 640090 words processed Wed Dec 23 15:37:14 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.468381. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50609 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.30768 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.37327 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.93919 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14206 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16244 clock time +[SCHEDULER ITER2 LR1]: 680075 words processed Wed Dec 23 15:37:31 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.469735. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50796 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.31745 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.39127 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.94572 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14593 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15991 clock time +[SCHEDULER ITER2 LR1]: 720043 words processed Wed Dec 23 15:37:48 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.469141. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50148 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.26346 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.26367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.89927 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13404 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15943 clock time +[SCHEDULER ITER2 LR1]: 760012 words processed Wed Dec 23 15:38:05 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.467519. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50113 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.29022 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.33786 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.93414 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13674 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15994 clock time +[SCHEDULER ITER2 LR1]: 800113 words processed Wed Dec 23 15:38:22 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.465771. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50477 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.33097 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.41780 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.95124 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15269 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16266 clock time +[SCHEDULER ITER2 LR1]: 840089 words processed Wed Dec 23 15:38:38 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.463527. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47668 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.04129 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.82154 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77515 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10621 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14924 clock time +[SCHEDULER ITER2 LR1]: 880052 words processed Wed Dec 23 15:38:54 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.461835. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47093 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99627 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74973 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76246 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10292 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14664 clock time [LOG]LMSeqReader: file expires, closing. +(15:38:57 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER2 LR1]: Displaying result: -[SCHEDULER ITER2 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 333.6037428687> <PPL_OOV 290.37375803538> <LOGP -2289538.074234> -[SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER2 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 332.64732402725> <PPL_OOV 289.55586875865> <LOGP -2288399.3336759> +[SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER2 LR1]: shuffling training file ===PEEK ON TEST 2=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER2 LR1]: 40087 words processed Wed Nov 18 10:18:18 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.380761. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46919 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74486 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.43873 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11428 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:38:59 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:38:59 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:38:59 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:38:59 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:38:59 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:38:59 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:38:59 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER2 LR1]: 40087 words processed Wed Dec 23 15:39:07 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.383660. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49089 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.16334 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.91126 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14168 clock time [LOG]LMSeqReader: file expires, closing. +(15:39:15 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER2 LR1]: Displaying result: -[SCHEDULER ITER2 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 275.16079647613> <PPL_OOV 242.77628129243> <LOGP -196612.55158828> +[SCHEDULER ITER2 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 282.14434885551> <PPL_OOV 244.8387619515> <LOGP -196915.39280752> [SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 2=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER2 LR1]: 40095 words processed Wed Nov 18 10:18:27 2015. - [SCHEDULER ITER2 LR1]: log prob per sample :-2.427169. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46685 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74609 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.43747 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11425 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:39:15 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:39:15 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:39:15 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:39:15 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:39:15 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:39:15 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:39:15 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER2 LR1]: 40095 words processed Wed Dec 23 15:39:23 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.432323. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49489 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.19944 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.95332 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14268 clock time [LOG]LMSeqReader: file expires, closing. +(15:39:29 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER2 LR1]: Displaying result: -[SCHEDULER ITER2 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 279.88256204919> <PPL_OOV 252.80637790871> <LOGP -177229.64482119> +[SCHEDULER ITER2 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 286.93385277057> <PPL_OOV 255.81525982596> <LOGP -177608.65475564> [SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER2 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.2... +[SCHEDULER ITER2 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.2... [SCHEDULER ITER3 LR1]: preparing parameters... -[SCHEDULER ITER3 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.2... +[SCHEDULER ITER3 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.2... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34525805 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3668 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34529462 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 119899 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69074656 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1196101 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69190773 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35751435 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70267310 +reading chunk 5 from 70280090 [SCHEDULER ITER3 LR1]: preparing parameters end. [SCHEDULER ITER3 LR1]: preparing layers... -(10:18:36 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:18:36 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:18:36 2015-11-18)[nerv] info: create layer: combinerL1 -(10:18:36 2015-11-18)[nerv] info: create layer: outputL -(10:18:36 2015-11-18)[nerv] info: create layer: softmaxL -(10:18:36 2015-11-18)[nerv] info: create layer: selectL1 +(15:39:35 2015-12-23)[nerv] info: create layer: sigmoidL1 +(15:39:35 2015-12-23)[nerv] info: create layer: combinerL1 +(15:39:35 2015-12-23)[nerv] info: create layer: selectL1 +(15:39:35 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(15:39:35 2015-12-23)[nerv] info: create layer: outputL +(15:39:35 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(15:39:35 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(15:39:35 2015-12-23)[nerv] info: create layer: softmaxL +(15:39:35 2015-12-23)[nerv] info: create layer: recurrentL1 +(15:39:35 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(15:39:35 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER3 LR1]: preparing layers end. [SCHEDULER ITER3 LR1]: Generate and initing TNN ... +(15:39:35 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -666,253 +749,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(15:39:35 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:39:35 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:39:35 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:39:35 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:39:35 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:39:35 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER3 LR1]: Initing TNN end. ===ITERATION 3 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER3 LR1]: 40092 words processed Wed Nov 18 10:18:47 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.536047. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46841 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72824 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22868 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56381 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10451 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11928 clock time -[SCHEDULER ITER3 LR1]: 80099 words processed Wed Nov 18 10:18:58 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.483923. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.71587 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.19287 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55022 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09750 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11941 clock time -[SCHEDULER ITER3 LR1]: 120004 words processed Wed Nov 18 10:19:10 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.458421. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46654 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.71644 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.18811 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54561 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09506 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11902 clock time -[SCHEDULER ITER3 LR1]: 160114 words processed Wed Nov 18 10:19:22 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.448869. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46564 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72314 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21041 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55762 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10313 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11887 clock time -[SCHEDULER ITER3 LR1]: 200066 words processed Wed Nov 18 10:19:34 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.441952. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46164 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72174 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.18569 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54325 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09779 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11905 clock time -[SCHEDULER ITER3 LR1]: 240045 words processed Wed Nov 18 10:19:46 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.432993. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46397 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72148 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.19430 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54829 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09770 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11958 clock time -[SCHEDULER ITER3 LR1]: 280057 words processed Wed Nov 18 10:19:58 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.427091. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46420 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72248 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22334 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56521 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10595 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11918 clock time -[SCHEDULER ITER3 LR1]: 320106 words processed Wed Nov 18 10:20:10 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.420076. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46241 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73121 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23973 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57037 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10789 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11974 clock time -[SCHEDULER ITER3 LR1]: 360024 words processed Wed Nov 18 10:20:22 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.416144. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46100 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72642 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20227 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55015 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09891 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11940 clock time -[SCHEDULER ITER3 LR1]: 400089 words processed Wed Nov 18 10:20:34 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.412820. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46143 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73168 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23198 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56644 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10677 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12013 clock time -[SCHEDULER ITER3 LR1]: 440067 words processed Wed Nov 18 10:20:46 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.410263. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46165 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72661 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.19516 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54854 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09580 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11913 clock time -[SCHEDULER ITER3 LR1]: 480051 words processed Wed Nov 18 10:20:58 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.407979. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72909 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20697 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55568 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09783 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11971 clock time -[SCHEDULER ITER3 LR1]: 520140 words processed Wed Nov 18 10:21:10 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.405734. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46027 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72708 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22339 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56616 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10415 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11944 clock time -[SCHEDULER ITER3 LR1]: 560132 words processed Wed Nov 18 10:21:22 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.403532. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45910 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.71769 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.18242 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54602 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09924 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11878 clock time -[SCHEDULER ITER3 LR1]: 600118 words processed Wed Nov 18 10:21:33 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.401175. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.71657 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.17742 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54693 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09537 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11830 clock time -[SCHEDULER ITER3 LR1]: 640090 words processed Wed Nov 18 10:21:45 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.398582. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45838 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.71715 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.17104 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54186 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09427 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11879 clock time -[SCHEDULER ITER3 LR1]: 680075 words processed Wed Nov 18 10:21:57 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.397244. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46118 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73236 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21270 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55627 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09701 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12054 clock time -[SCHEDULER ITER3 LR1]: 720043 words processed Wed Nov 18 10:22:09 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.396091. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46029 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72831 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20086 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55243 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09483 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11977 clock time -[SCHEDULER ITER3 LR1]: 760012 words processed Wed Nov 18 10:22:21 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.394789. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46249 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72900 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20841 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55161 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10074 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11922 clock time -[SCHEDULER ITER3 LR1]: 800113 words processed Wed Nov 18 10:22:33 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.393428. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45947 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72019 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20872 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56109 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10514 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11855 clock time -[SCHEDULER ITER3 LR1]: 840089 words processed Wed Nov 18 10:22:45 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.391343. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.71939 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.17808 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54550 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09538 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11851 clock time -[SCHEDULER ITER3 LR1]: 880052 words processed Wed Nov 18 10:22:57 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.389966. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45967 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72022 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.18124 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.54495 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09355 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11946 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:39:35 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:39:35 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:39:35 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:39:35 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:39:35 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:39:35 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:39:35 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER3 LR1]: 40092 words processed Wed Dec 23 15:39:51 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.420985. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48706 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.03868 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.85745 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78809 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11542 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15019 clock time +[SCHEDULER ITER3 LR1]: 80099 words processed Wed Dec 23 15:40:07 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.418875. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48461 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.01189 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.80739 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76993 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11032 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15114 clock time +[SCHEDULER ITER3 LR1]: 120004 words processed Wed Dec 23 15:40:23 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.410982. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47899 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97154 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71601 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74641 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09915 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14682 clock time +[SCHEDULER ITER3 LR1]: 160114 words processed Wed Dec 23 15:40:39 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.410298. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47868 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98721 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74936 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76177 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10479 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14748 clock time +[SCHEDULER ITER3 LR1]: 200066 words processed Wed Dec 23 15:40:55 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.408833. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47905 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00940 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.78087 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76505 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10448 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14922 clock time +[SCHEDULER ITER3 LR1]: 240045 words processed Wed Dec 23 15:41:11 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.402712. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47642 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98880 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74640 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75689 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10148 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14914 clock time +[SCHEDULER ITER3 LR1]: 280057 words processed Wed Dec 23 15:41:27 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.399028. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48106 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.02661 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.83337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78423 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11002 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15012 clock time +[SCHEDULER ITER3 LR1]: 320106 words processed Wed Dec 23 15:41:43 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.394132. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.01093 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.81025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78432 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10945 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15013 clock time +[SCHEDULER ITER3 LR1]: 360024 words processed Wed Dec 23 15:41:59 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.391508. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47224 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98878 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73757 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75422 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09910 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14825 clock time +[SCHEDULER ITER3 LR1]: 400089 words processed Wed Dec 23 15:42:15 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.389884. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47585 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.03087 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.84521 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.79220 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11260 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15172 clock time +[SCHEDULER ITER3 LR1]: 440067 words processed Wed Dec 23 15:42:32 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.388327. + [global_conf.timer]: time spent on tnn_beforeprocess:0.51026 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.38877 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.53851 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.97731 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15614 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16682 clock time +[SCHEDULER ITER3 LR1]: 480051 words processed Wed Dec 23 15:42:48 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.387093. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48494 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.11183 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.97720 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.81591 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11867 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15559 clock time +[SCHEDULER ITER3 LR1]: 520140 words processed Wed Dec 23 15:43:04 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.385775. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48538 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.10118 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.97996 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.82319 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12262 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15605 clock time +[SCHEDULER ITER3 LR1]: 560132 words processed Wed Dec 23 15:43:20 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.384376. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48608 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.09976 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.97588 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.82639 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11800 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15594 clock time +[SCHEDULER ITER3 LR1]: 600118 words processed Wed Dec 23 15:43:36 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.383014. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48815 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.10608 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.98839 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.82429 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12069 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15797 clock time +[SCHEDULER ITER3 LR1]: 640090 words processed Wed Dec 23 15:43:52 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.381368. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48797 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.09279 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.95575 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.81537 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11474 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15599 clock time +[SCHEDULER ITER3 LR1]: 680075 words processed Wed Dec 23 15:44:08 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.380063. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.05583 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.90136 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.80839 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11586 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15433 clock time +[SCHEDULER ITER3 LR1]: 720043 words processed Wed Dec 23 15:44:24 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.379232. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48700 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.09829 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.95980 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.80995 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11736 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15596 clock time +[SCHEDULER ITER3 LR1]: 760012 words processed Wed Dec 23 15:44:40 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.378130. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47789 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.06108 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.89730 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.80026 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11489 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15442 clock time +[SCHEDULER ITER3 LR1]: 800113 words processed Wed Dec 23 15:44:56 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.377008. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48403 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.11655 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.01752 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.83852 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12848 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15569 clock time +[SCHEDULER ITER3 LR1]: 840089 words processed Wed Dec 23 15:45:12 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.375355. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48865 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12383 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.00665 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.82478 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12018 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15772 clock time +[SCHEDULER ITER3 LR1]: 880052 words processed Wed Dec 23 15:45:28 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.374309. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48280 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.09057 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.94255 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.80734 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11672 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15489 clock time [LOG]LMSeqReader: file expires, closing. +(15:45:32 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER3 LR1]: Displaying result: -[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 280.05523911791> <PPL_OOV 245.39663556928> <LOGP -2221595.5658898> -[SCHEDULER ITER3 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 269.79300306403> <PPL_OOV 236.75448749157> <LOGP -2207121.507527> +[SCHEDULER ITER3 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER3 LR1]: shuffling training file ===PEEK ON TEST 3=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER3 LR1]: 40087 words processed Wed Nov 18 10:23:06 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.338319. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46632 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73173 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.42069 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11427 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:45:33 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:45:33 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:45:33 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:45:33 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:45:33 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:45:33 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:45:33 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER3 LR1]: 40087 words processed Wed Dec 23 15:45:41 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.332513. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50703 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.28113 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:7.06561 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15139 clock time [LOG]LMSeqReader: file expires, closing. +(15:45:49 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER3 LR1]: Displaying result: -[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 249.21738984732> <PPL_OOV 218.98102897851> <LOGP -192919.70714879> +[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 249.02175046571> <PPL_OOV 216.08139951757> <LOGP -192442.51096561> [SCHEDULER ITER3 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 3=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER3 LR1]: 40095 words processed Wed Nov 18 10:23:15 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.387513. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46562 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73144 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.41963 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11445 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:45:49 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:45:49 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:45:49 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:45:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:45:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:45:49 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:45:49 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER3 LR1]: 40095 words processed Wed Dec 23 15:45:57 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.386586. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50295 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.28988 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:7.06698 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14933 clock time [LOG]LMSeqReader: file expires, closing. +(15:46:04 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER3 LR1]: Displaying result: -[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 253.6486382493> <PPL_OOV 228.35790895447> <LOGP -173971.52040645> +[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 253.7213160687> <PPL_OOV 226.30462675741> <LOGP -173682.18762679> [SCHEDULER ITER3 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER3 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.3... +[SCHEDULER ITER3 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.3... [SCHEDULER ITER4 LR1]: preparing parameters... -[SCHEDULER ITER4 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.3... +[SCHEDULER ITER4 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.3... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1076112 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3667 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1079769 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 119924 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35639160 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1195723 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35755265 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35765487 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70285481 +reading chunk 5 from 70299639 [SCHEDULER ITER4 LR1]: preparing parameters end. [SCHEDULER ITER4 LR1]: preparing layers... -(10:23:24 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:23:24 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:23:24 2015-11-18)[nerv] info: create layer: combinerL1 -(10:23:24 2015-11-18)[nerv] info: create layer: outputL -(10:23:24 2015-11-18)[nerv] info: create layer: softmaxL -(10:23:24 2015-11-18)[nerv] info: create layer: selectL1 +(15:46:09 2015-12-23)[nerv] info: create layer: sigmoidL1 +(15:46:09 2015-12-23)[nerv] info: create layer: combinerL1 +(15:46:09 2015-12-23)[nerv] info: create layer: selectL1 +(15:46:09 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(15:46:09 2015-12-23)[nerv] info: create layer: outputL +(15:46:09 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(15:46:09 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(15:46:09 2015-12-23)[nerv] info: create layer: softmaxL +(15:46:09 2015-12-23)[nerv] info: create layer: recurrentL1 +(15:46:09 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(15:46:09 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER4 LR1]: preparing layers end. [SCHEDULER ITER4 LR1]: Generate and initing TNN ... +(15:46:09 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -928,253 +1047,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(15:46:09 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:46:09 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:46:09 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:46:09 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:46:09 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:46:09 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER4 LR1]: Initing TNN end. ===ITERATION 4 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER4 LR1]: 40092 words processed Wed Nov 18 10:23:35 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.356761. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47160 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75773 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.29264 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58409 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10959 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11998 clock time -[SCHEDULER ITER4 LR1]: 80099 words processed Wed Nov 18 10:23:47 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.354343. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46970 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73607 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21454 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55435 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09187 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11917 clock time -[SCHEDULER ITER4 LR1]: 120004 words processed Wed Nov 18 10:23:59 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.348934. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46688 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73326 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21025 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55149 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09277 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11852 clock time -[SCHEDULER ITER4 LR1]: 160114 words processed Wed Nov 18 10:24:11 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.350011. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47103 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76141 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.29215 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58396 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10589 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12027 clock time -[SCHEDULER ITER4 LR1]: 200066 words processed Wed Nov 18 10:24:23 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.350221. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46486 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74205 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22571 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09508 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11912 clock time -[SCHEDULER ITER4 LR1]: 240045 words processed Wed Nov 18 10:24:35 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.346361. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46475 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75070 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24038 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56038 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09840 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11952 clock time -[SCHEDULER ITER4 LR1]: 280057 words processed Wed Nov 18 10:24:47 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.343892. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46855 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75436 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27547 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57787 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10433 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11990 clock time -[SCHEDULER ITER4 LR1]: 320106 words processed Wed Nov 18 10:24:59 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.340016. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46557 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75534 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27682 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58157 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10343 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12000 clock time -[SCHEDULER ITER4 LR1]: 360024 words processed Wed Nov 18 10:25:11 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.338481. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46402 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75085 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25085 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10167 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11995 clock time -[SCHEDULER ITER4 LR1]: 400089 words processed Wed Nov 18 10:25:23 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.336970. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46259 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75447 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26929 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10384 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11980 clock time -[SCHEDULER ITER4 LR1]: 440067 words processed Wed Nov 18 10:25:35 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.336232. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46305 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74314 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22687 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55912 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09659 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11915 clock time -[SCHEDULER ITER4 LR1]: 480051 words processed Wed Nov 18 10:25:47 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.335499. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46142 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74277 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22475 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55799 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09857 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11955 clock time -[SCHEDULER ITER4 LR1]: 520140 words processed Wed Nov 18 10:25:59 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.334860. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46425 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75886 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27816 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58048 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10432 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12038 clock time -[SCHEDULER ITER4 LR1]: 560132 words processed Wed Nov 18 10:26:11 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.333952. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46205 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74473 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22480 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56016 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09360 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11929 clock time -[SCHEDULER ITER4 LR1]: 600118 words processed Wed Nov 18 10:26:23 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.332777. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46069 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74222 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21981 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55734 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09515 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11931 clock time -[SCHEDULER ITER4 LR1]: 640090 words processed Wed Nov 18 10:26:35 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.331373. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46163 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74294 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22136 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09495 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11914 clock time -[SCHEDULER ITER4 LR1]: 680075 words processed Wed Nov 18 10:26:47 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.330378. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46080 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73936 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22054 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55849 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09770 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11932 clock time -[SCHEDULER ITER4 LR1]: 720043 words processed Wed Nov 18 10:26:59 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.329916. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73811 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20900 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55371 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09249 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11881 clock time -[SCHEDULER ITER4 LR1]: 760012 words processed Wed Nov 18 10:27:11 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.329256. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46228 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74636 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23114 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55924 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09538 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11952 clock time -[SCHEDULER ITER4 LR1]: 800113 words processed Wed Nov 18 10:27:23 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.328670. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46365 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75632 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27123 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57626 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10571 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11989 clock time -[SCHEDULER ITER4 LR1]: 840089 words processed Wed Nov 18 10:27:35 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.327835. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46185 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74296 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22645 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56132 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09518 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11956 clock time -[SCHEDULER ITER4 LR1]: 880052 words processed Wed Nov 18 10:27:47 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.327304. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46200 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75015 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24761 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56572 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10060 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12040 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:46:09 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:46:09 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:46:09 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:46:09 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:46:09 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:46:09 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:46:09 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER4 LR1]: 40092 words processed Wed Dec 23 15:46:25 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.608697. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49827 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12206 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.04130 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.84225 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13536 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15645 clock time +[SCHEDULER ITER4 LR1]: 80099 words processed Wed Dec 23 15:46:42 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.510703. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50195 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.14228 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.08100 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.85191 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13104 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16005 clock time +[SCHEDULER ITER4 LR1]: 120004 words processed Wed Dec 23 15:46:58 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.468678. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49208 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.10009 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.97520 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.81684 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12170 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15561 clock time +[SCHEDULER ITER4 LR1]: 160114 words processed Wed Dec 23 15:47:14 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.448771. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49534 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.13373 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.06151 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.85147 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13420 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15823 clock time +[SCHEDULER ITER4 LR1]: 200066 words processed Wed Dec 23 15:47:31 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.435165. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49502 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.15309 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.09227 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.85978 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13035 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15824 clock time +[SCHEDULER ITER4 LR1]: 240045 words processed Wed Dec 23 15:47:47 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.420254. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47699 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99081 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75632 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76665 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10476 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14824 clock time +[SCHEDULER ITER4 LR1]: 280057 words processed Wed Dec 23 15:48:04 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.409666. + [global_conf.timer]: time spent on tnn_beforeprocess:0.52410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.36777 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.57103 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:6.00884 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.17159 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16786 clock time +[SCHEDULER ITER4 LR1]: 320106 words processed Wed Dec 23 15:48:20 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.399763. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48092 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.08781 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.94315 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.81554 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12297 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15302 clock time +[SCHEDULER ITER4 LR1]: 360024 words processed Wed Dec 23 15:48:37 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.393229. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50328 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.24452 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:15.25886 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.89674 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14321 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.16275 clock time +[SCHEDULER ITER4 LR1]: 400089 words processed Wed Dec 23 15:48:53 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.388415. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47827 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.05504 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.89224 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.81175 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11716 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15243 clock time +[SCHEDULER ITER4 LR1]: 440067 words processed Wed Dec 23 15:49:09 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.384129. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46942 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96590 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70437 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75530 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10146 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14770 clock time +[SCHEDULER ITER4 LR1]: 480051 words processed Wed Dec 23 15:49:25 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.380379. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46997 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96224 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68887 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74705 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09947 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14710 clock time +[SCHEDULER ITER4 LR1]: 520140 words processed Wed Dec 23 15:49:41 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.377087. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47487 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.02414 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.82861 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.79097 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11442 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14960 clock time +[SCHEDULER ITER4 LR1]: 560132 words processed Wed Dec 23 15:49:57 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.373787. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47062 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97778 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71807 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75479 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10326 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14686 clock time +[SCHEDULER ITER4 LR1]: 600118 words processed Wed Dec 23 15:50:13 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.370727. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47717 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.03144 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.82422 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78184 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11269 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15037 clock time +[SCHEDULER ITER4 LR1]: 640090 words processed Wed Dec 23 15:50:29 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.367257. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.03055 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.81958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78111 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10952 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15113 clock time +[SCHEDULER ITER4 LR1]: 680075 words processed Wed Dec 23 15:50:45 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.364960. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47069 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97440 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71875 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75786 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10225 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14748 clock time +[SCHEDULER ITER4 LR1]: 720043 words processed Wed Dec 23 15:51:01 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.362891. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46960 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96683 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70265 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75183 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10195 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14663 clock time +[SCHEDULER ITER4 LR1]: 760012 words processed Wed Dec 23 15:51:17 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.360646. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46909 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96630 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69458 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74867 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09892 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14653 clock time +[SCHEDULER ITER4 LR1]: 800113 words processed Wed Dec 23 15:51:33 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.358760. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47082 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98754 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74993 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76948 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10851 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14708 clock time +[SCHEDULER ITER4 LR1]: 840089 words processed Wed Dec 23 15:51:49 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.356208. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46887 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96581 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69187 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09959 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14646 clock time +[SCHEDULER ITER4 LR1]: 880052 words processed Wed Dec 23 15:52:05 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.354538. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46956 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96705 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70075 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75024 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10148 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14699 clock time [LOG]LMSeqReader: file expires, closing. +(15:52:08 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER4 LR1]: Displaying result: -[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 241.20958994184> <PPL_OOV 212.5016612632> <LOGP -2163490.4205743> -[SCHEDULER ITER4 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 256.37905489956> <PPL_OOV 226.16752624328> <LOGP -2188652.4596846> +[SCHEDULER ITER4 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER4 LR1]: shuffling training file ===PEEK ON TEST 4=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER4 LR1]: 40087 words processed Wed Nov 18 10:27:56 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.311922. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47680 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.79140 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.50005 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11768 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:52:10 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:52:10 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:52:10 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:52:10 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:52:10 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:52:10 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:52:10 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER4 LR1]: 40087 words processed Wed Dec 23 15:52:18 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.306634. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49333 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17613 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92977 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14349 clock time [LOG]LMSeqReader: file expires, closing. +(15:52:26 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER4 LR1]: Displaying result: -[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 229.16655843007> <PPL_OOV 205.79109682218> <LOGP -190695.75314365> +[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 234.24814652203> <PPL_OOV 203.64078808> <LOGP -190319.72291421> [SCHEDULER ITER4 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 4=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER4 LR1]: 40095 words processed Wed Nov 18 10:28:05 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.361644. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46707 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74527 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.43673 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11440 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:52:26 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:52:26 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:52:26 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:52:26 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:52:26 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:52:26 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:52:26 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER4 LR1]: 40095 words processed Wed Dec 23 15:52:34 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.361597. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49800 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.24906 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:7.01591 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14676 clock time [LOG]LMSeqReader: file expires, closing. +(15:52:40 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER4 LR1]: Displaying result: -[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 233.67513484246> <PPL_OOV 214.05170779862> <LOGP -171899.05910373> +[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 239.9440237225> <PPL_OOV 214.10854239161> <LOGP -171907.56346354> [SCHEDULER ITER4 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER4 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.4... +[SCHEDULER ITER4 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.4... [SCHEDULER ITER5 LR1]: preparing parameters... -[SCHEDULER ITER5 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.4... +[SCHEDULER ITER5 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.4... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34533992 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3670 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34537650 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 119979 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69107906 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1195345 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69224062 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35778230 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70299750 +reading chunk 5 from 70316633 [SCHEDULER ITER5 LR1]: preparing parameters end. [SCHEDULER ITER5 LR1]: preparing layers... -(10:28:14 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:28:14 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:28:14 2015-11-18)[nerv] info: create layer: combinerL1 -(10:28:14 2015-11-18)[nerv] info: create layer: outputL -(10:28:14 2015-11-18)[nerv] info: create layer: softmaxL -(10:28:14 2015-11-18)[nerv] info: create layer: selectL1 +(15:52:46 2015-12-23)[nerv] info: create layer: sigmoidL1 +(15:52:46 2015-12-23)[nerv] info: create layer: combinerL1 +(15:52:46 2015-12-23)[nerv] info: create layer: selectL1 +(15:52:46 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(15:52:46 2015-12-23)[nerv] info: create layer: outputL +(15:52:46 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(15:52:46 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(15:52:46 2015-12-23)[nerv] info: create layer: softmaxL +(15:52:46 2015-12-23)[nerv] info: create layer: recurrentL1 +(15:52:46 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(15:52:46 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER5 LR1]: preparing layers end. [SCHEDULER ITER5 LR1]: Generate and initing TNN ... +(15:52:46 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -1190,253 +1345,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(15:52:46 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:52:46 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:52:46 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:52:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:52:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:52:46 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER5 LR1]: Initing TNN end. ===ITERATION 5 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER5 LR1]: 40092 words processed Wed Nov 18 10:28:25 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.342229. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47049 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75456 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.28763 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58442 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11110 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11981 clock time -[SCHEDULER ITER5 LR1]: 80099 words processed Wed Nov 18 10:28:37 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.325496. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46939 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74216 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22949 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55726 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09820 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11913 clock time -[SCHEDULER ITER5 LR1]: 120004 words processed Wed Nov 18 10:28:49 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.314607. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46910 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74325 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23677 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55937 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09786 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11937 clock time -[SCHEDULER ITER5 LR1]: 160114 words processed Wed Nov 18 10:29:01 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.314235. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47069 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75472 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27042 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57500 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10372 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12016 clock time -[SCHEDULER ITER5 LR1]: 200066 words processed Wed Nov 18 10:29:13 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.312322. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46407 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73731 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22218 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55908 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09757 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11939 clock time -[SCHEDULER ITER5 LR1]: 240045 words processed Wed Nov 18 10:29:25 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.308654. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46513 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74426 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23656 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55966 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10261 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11947 clock time -[SCHEDULER ITER5 LR1]: 280057 words processed Wed Nov 18 10:29:37 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.305328. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46782 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75094 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26266 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57342 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10246 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11961 clock time -[SCHEDULER ITER5 LR1]: 320106 words processed Wed Nov 18 10:29:49 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.301431. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75036 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26305 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57695 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10340 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11993 clock time -[SCHEDULER ITER5 LR1]: 360024 words processed Wed Nov 18 10:30:01 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.299609. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74390 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23802 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56393 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09906 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11951 clock time -[SCHEDULER ITER5 LR1]: 400089 words processed Wed Nov 18 10:30:13 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.297780. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46554 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75843 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.28051 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57991 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10791 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12014 clock time -[SCHEDULER ITER5 LR1]: 440067 words processed Wed Nov 18 10:30:25 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.297017. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46181 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74817 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23907 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56744 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09555 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12009 clock time -[SCHEDULER ITER5 LR1]: 480051 words processed Wed Nov 18 10:30:37 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.296471. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46227 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74135 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23129 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56201 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10193 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11990 clock time -[SCHEDULER ITER5 LR1]: 520140 words processed Wed Nov 18 10:30:49 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.295813. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46426 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75114 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26550 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57283 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11095 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11931 clock time -[SCHEDULER ITER5 LR1]: 560132 words processed Wed Nov 18 10:31:01 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.294844. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46199 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74167 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22088 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56236 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09129 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11941 clock time -[SCHEDULER ITER5 LR1]: 600118 words processed Wed Nov 18 10:31:13 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.296112. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46173 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74829 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24560 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56703 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10230 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12016 clock time -[SCHEDULER ITER5 LR1]: 640090 words processed Wed Nov 18 10:31:25 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.294628. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46158 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74747 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24467 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56448 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10528 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11997 clock time -[SCHEDULER ITER5 LR1]: 680075 words processed Wed Nov 18 10:31:37 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.294433. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46073 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73934 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21765 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56040 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09367 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11932 clock time -[SCHEDULER ITER5 LR1]: 720043 words processed Wed Nov 18 10:31:49 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.294059. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46088 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74279 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23473 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56187 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10443 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11920 clock time -[SCHEDULER ITER5 LR1]: 760012 words processed Wed Nov 18 10:32:01 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.293522. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46137 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74359 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23332 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56142 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10053 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12005 clock time -[SCHEDULER ITER5 LR1]: 800113 words processed Wed Nov 18 10:32:13 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.293090. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46287 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74776 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25004 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57351 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09928 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11979 clock time -[SCHEDULER ITER5 LR1]: 840089 words processed Wed Nov 18 10:32:25 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.291937. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46324 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74901 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24348 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56479 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10076 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11998 clock time -[SCHEDULER ITER5 LR1]: 880052 words processed Wed Nov 18 10:32:37 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.291464. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46515 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74681 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22951 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56141 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09092 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11908 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:52:46 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:52:46 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:52:46 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:52:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:52:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:52:46 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:52:46 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER5 LR1]: 40092 words processed Wed Dec 23 15:53:02 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.319916. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48931 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.06315 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.92985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.81857 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12141 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15281 clock time +[SCHEDULER ITER5 LR1]: 80099 words processed Wed Dec 23 15:53:18 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.320699. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48269 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99266 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76975 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76847 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10221 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15020 clock time +[SCHEDULER ITER5 LR1]: 120004 words processed Wed Dec 23 15:53:34 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.314060. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48190 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00488 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.79825 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77456 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10703 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15077 clock time +[SCHEDULER ITER5 LR1]: 160114 words processed Wed Dec 23 15:53:50 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.315198. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48475 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.05139 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.89513 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.80915 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11532 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15344 clock time +[SCHEDULER ITER5 LR1]: 200066 words processed Wed Dec 23 15:54:06 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.315035. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47754 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00197 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.77739 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77479 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10123 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14895 clock time +[SCHEDULER ITER5 LR1]: 240045 words processed Wed Dec 23 15:54:22 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.309852. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47546 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98386 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73550 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76048 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09942 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14691 clock time +[SCHEDULER ITER5 LR1]: 280057 words processed Wed Dec 23 15:54:38 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.306431. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47554 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98696 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76230 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77774 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10277 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14720 clock time +[SCHEDULER ITER5 LR1]: 320106 words processed Wed Dec 23 15:54:54 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.302927. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47360 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99237 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.77140 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77850 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10726 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14758 clock time +[SCHEDULER ITER5 LR1]: 360024 words processed Wed Dec 23 15:55:10 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.301453. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96756 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70960 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75715 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09717 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14722 clock time +[SCHEDULER ITER5 LR1]: 400089 words processed Wed Dec 23 15:55:26 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.300367. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47429 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.01549 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.80429 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78655 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10692 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14890 clock time +[SCHEDULER ITER5 LR1]: 440067 words processed Wed Dec 23 15:55:42 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.299777. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46963 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96520 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70068 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75500 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09685 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14687 clock time +[SCHEDULER ITER5 LR1]: 480051 words processed Wed Dec 23 15:55:58 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.299221. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46942 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96602 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69701 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75334 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09529 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14696 clock time +[SCHEDULER ITER5 LR1]: 520140 words processed Wed Dec 23 15:56:14 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.298647. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47141 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99228 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.77247 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78164 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10690 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14803 clock time +[SCHEDULER ITER5 LR1]: 560132 words processed Wed Dec 23 15:56:30 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.297849. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46943 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96554 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75793 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09707 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14743 clock time +[SCHEDULER ITER5 LR1]: 600118 words processed Wed Dec 23 15:56:46 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.295752. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47202 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98611 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73446 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76033 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09923 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14779 clock time +[SCHEDULER ITER5 LR1]: 640090 words processed Wed Dec 23 15:57:02 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.295468. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46955 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96842 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70506 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75547 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09683 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14721 clock time +[SCHEDULER ITER5 LR1]: 680075 words processed Wed Dec 23 15:57:18 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.294879. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47711 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.03788 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.82968 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78424 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10651 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15047 clock time +[SCHEDULER ITER5 LR1]: 720043 words processed Wed Dec 23 15:57:34 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.294478. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46883 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96217 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69610 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75335 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09663 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14693 clock time +[SCHEDULER ITER5 LR1]: 760012 words processed Wed Dec 23 15:57:50 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.293772. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46901 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96227 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69930 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75424 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09805 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14730 clock time +[SCHEDULER ITER5 LR1]: 800113 words processed Wed Dec 23 15:58:06 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.293194. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47157 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98431 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75312 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77559 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10408 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14742 clock time +[SCHEDULER ITER5 LR1]: 840089 words processed Wed Dec 23 15:58:22 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.292008. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46945 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96345 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69444 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75198 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09601 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14672 clock time +[SCHEDULER ITER5 LR1]: 880052 words processed Wed Dec 23 15:58:38 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.291458. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46903 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96239 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69665 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75390 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09522 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time [LOG]LMSeqReader: file expires, closing. +(15:58:41 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER5 LR1]: Displaying result: -[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 221.30733235924> <PPL_OOV 195.66770349262> <LOGP -2130170.993105> -[SCHEDULER ITER5 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 221.22448597638> <PPL_OOV 195.66253907145> <LOGP -2130160.3373672> +[SCHEDULER ITER5 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER5 LR1]: shuffling training file ===PEEK ON TEST 5=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER5 LR1]: 40087 words processed Wed Nov 18 10:32:46 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.285627. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46841 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75339 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.44481 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11435 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:58:43 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:58:43 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:58:43 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:58:43 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:58:43 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:58:43 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:58:43 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER5 LR1]: 40087 words processed Wed Dec 23 15:58:51 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.282752. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49175 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.14921 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.90305 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14309 clock time [LOG]LMSeqReader: file expires, closing. +(15:58:59 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER5 LR1]: Displaying result: -[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 222.8321336645> <PPL_OOV 193.91842079921> <LOGP -188568.43959101> +[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 220.22987876153> <PPL_OOV 192.30415908048> <LOGP -188269.18665288> [SCHEDULER ITER5 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 5=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER5 LR1]: 40095 words processed Wed Nov 18 10:32:55 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.339220. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46835 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75101 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.44223 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11451 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:58:59 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:58:59 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:58:59 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:58:59 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:58:59 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:58:59 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:58:59 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER5 LR1]: 40095 words processed Wed Dec 23 15:59:07 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.338530. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49358 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.20145 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.95702 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14326 clock time [LOG]LMSeqReader: file expires, closing. +(15:59:13 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER5 LR1]: Displaying result: -[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 228.12299970668> <PPL_OOV 203.70434101172> <LOGP -170311.86095287> +[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 226.01358239564> <PPL_OOV 202.45746765523> <LOGP -170115.18111578> [SCHEDULER ITER5 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER5 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.5... +[SCHEDULER ITER5 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.5... [SCHEDULER ITER6 LR1]: preparing parameters... -[SCHEDULER ITER6 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.5... +[SCHEDULER ITER6 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.5... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1075225 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3669 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1078883 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120042 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35657664 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1194944 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35773857 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35787768 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70310639 +reading chunk 5 from 70329136 [SCHEDULER ITER6 LR1]: preparing parameters end. [SCHEDULER ITER6 LR1]: preparing layers... -(10:33:04 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:33:04 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:33:04 2015-11-18)[nerv] info: create layer: combinerL1 -(10:33:04 2015-11-18)[nerv] info: create layer: outputL -(10:33:04 2015-11-18)[nerv] info: create layer: softmaxL -(10:33:04 2015-11-18)[nerv] info: create layer: selectL1 +(15:59:19 2015-12-23)[nerv] info: create layer: sigmoidL1 +(15:59:19 2015-12-23)[nerv] info: create layer: combinerL1 +(15:59:19 2015-12-23)[nerv] info: create layer: selectL1 +(15:59:19 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(15:59:19 2015-12-23)[nerv] info: create layer: outputL +(15:59:19 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(15:59:19 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(15:59:19 2015-12-23)[nerv] info: create layer: softmaxL +(15:59:19 2015-12-23)[nerv] info: create layer: recurrentL1 +(15:59:19 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(15:59:19 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER6 LR1]: preparing layers end. [SCHEDULER ITER6 LR1]: Generate and initing TNN ... +(15:59:19 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -1452,253 +1643,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(15:59:19 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:59:19 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:59:19 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:59:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:59:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:59:19 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER6 LR1]: Initing TNN end. ===ITERATION 6 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER6 LR1]: 40092 words processed Wed Nov 18 10:33:15 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.274624. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46939 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75387 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.28249 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58495 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10652 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11975 clock time -[SCHEDULER ITER6 LR1]: 80099 words processed Wed Nov 18 10:33:27 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.274089. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46998 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74679 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24110 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09641 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11948 clock time -[SCHEDULER ITER6 LR1]: 120004 words processed Wed Nov 18 10:33:39 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.269376. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46942 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74932 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25240 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56915 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09510 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11974 clock time -[SCHEDULER ITER6 LR1]: 160114 words processed Wed Nov 18 10:33:51 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.272529. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46932 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75383 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27805 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58302 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10538 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11978 clock time -[SCHEDULER ITER6 LR1]: 200066 words processed Wed Nov 18 10:34:03 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.273086. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46566 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74590 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24741 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56897 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09888 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11994 clock time -[SCHEDULER ITER6 LR1]: 240045 words processed Wed Nov 18 10:34:15 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.270336. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46523 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74558 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24194 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56707 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09739 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11960 clock time -[SCHEDULER ITER6 LR1]: 280057 words processed Wed Nov 18 10:34:27 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.267457. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46955 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76631 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.30883 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59048 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10752 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12089 clock time -[SCHEDULER ITER6 LR1]: 320106 words processed Wed Nov 18 10:34:39 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.264325. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46362 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75084 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26563 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57789 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10421 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11986 clock time -[SCHEDULER ITER6 LR1]: 360024 words processed Wed Nov 18 10:34:51 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.263165. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73936 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21825 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09307 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11887 clock time -[SCHEDULER ITER6 LR1]: 400089 words processed Wed Nov 18 10:35:03 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.261798. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46289 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75369 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27194 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58193 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10492 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12021 clock time -[SCHEDULER ITER6 LR1]: 440067 words processed Wed Nov 18 10:35:15 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.261720. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46106 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74338 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23310 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56478 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09902 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11938 clock time -[SCHEDULER ITER6 LR1]: 480051 words processed Wed Nov 18 10:35:27 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.261638. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46115 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74888 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24538 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56905 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10084 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11952 clock time -[SCHEDULER ITER6 LR1]: 520140 words processed Wed Nov 18 10:35:39 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.261379. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46261 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75682 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27595 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58229 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10587 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11963 clock time -[SCHEDULER ITER6 LR1]: 560132 words processed Wed Nov 18 10:35:51 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.260855. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46142 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75126 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25299 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56931 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10505 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12021 clock time -[SCHEDULER ITER6 LR1]: 600118 words processed Wed Nov 18 10:36:03 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.259417. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46378 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75487 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26827 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57611 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10447 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12054 clock time -[SCHEDULER ITER6 LR1]: 640090 words processed Wed Nov 18 10:36:15 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.258156. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46052 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74638 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23792 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56574 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10022 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11938 clock time -[SCHEDULER ITER6 LR1]: 680075 words processed Wed Nov 18 10:36:27 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.257716. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46070 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74478 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23722 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56557 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10165 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11943 clock time -[SCHEDULER ITER6 LR1]: 720043 words processed Wed Nov 18 10:36:39 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.257695. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46112 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75192 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24788 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56738 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10015 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11978 clock time -[SCHEDULER ITER6 LR1]: 760012 words processed Wed Nov 18 10:36:51 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.257372. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46108 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74870 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25056 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57122 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10129 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12031 clock time -[SCHEDULER ITER6 LR1]: 800113 words processed Wed Nov 18 10:37:03 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.257342. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46458 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76279 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.29298 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58622 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10878 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12045 clock time -[SCHEDULER ITER6 LR1]: 840089 words processed Wed Nov 18 10:37:15 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.256451. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46283 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75097 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24796 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57053 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09715 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11975 clock time -[SCHEDULER ITER6 LR1]: 880052 words processed Wed Nov 18 10:37:27 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.256282. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46175 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74750 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24244 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56506 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10116 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11972 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(15:59:19 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(15:59:19 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(15:59:19 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(15:59:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(15:59:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(15:59:19 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(15:59:19 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER6 LR1]: 40092 words processed Wed Dec 23 15:59:35 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.282294. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48182 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00431 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.79515 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78076 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11125 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14831 clock time +[SCHEDULER ITER6 LR1]: 80099 words processed Wed Dec 23 15:59:51 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.280524. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47924 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95849 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69237 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74630 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09907 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14652 clock time +[SCHEDULER ITER6 LR1]: 120004 words processed Wed Dec 23 16:00:07 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.275626. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47736 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96168 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70573 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75244 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09946 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14670 clock time +[SCHEDULER ITER6 LR1]: 160114 words processed Wed Dec 23 16:00:23 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.276758. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47852 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99722 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.78486 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78083 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11068 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14809 clock time +[SCHEDULER ITER6 LR1]: 200066 words processed Wed Dec 23 16:00:39 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.277294. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47342 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96521 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71018 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75660 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10142 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14681 clock time +[SCHEDULER ITER6 LR1]: 240045 words processed Wed Dec 23 16:00:55 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.272584. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47356 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96034 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69040 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74799 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09856 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14641 clock time +[SCHEDULER ITER6 LR1]: 280057 words processed Wed Dec 23 16:01:11 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.269537. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47539 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98233 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75283 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77199 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10610 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14789 clock time +[SCHEDULER ITER6 LR1]: 320106 words processed Wed Dec 23 16:01:27 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.266237. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47319 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98591 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75681 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77499 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10552 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14796 clock time +[SCHEDULER ITER6 LR1]: 360024 words processed Wed Dec 23 16:01:43 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.265110. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46969 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96381 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69951 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75190 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09944 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14696 clock time +[SCHEDULER ITER6 LR1]: 400089 words processed Wed Dec 23 16:01:59 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.263660. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47078 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98218 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74144 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76912 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10481 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14738 clock time +[SCHEDULER ITER6 LR1]: 440067 words processed Wed Dec 23 16:02:15 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.263304. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47125 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98988 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74428 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76596 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10246 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14775 clock time +[SCHEDULER ITER6 LR1]: 480051 words processed Wed Dec 23 16:02:31 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.262897. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46853 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96046 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68861 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75049 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09945 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14696 clock time +[SCHEDULER ITER6 LR1]: 520140 words processed Wed Dec 23 16:02:47 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.262629. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47008 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98166 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74061 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76893 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10577 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14728 clock time +[SCHEDULER ITER6 LR1]: 560132 words processed Wed Dec 23 16:03:03 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.261979. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46969 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97292 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70720 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75456 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09842 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14766 clock time +[SCHEDULER ITER6 LR1]: 600118 words processed Wed Dec 23 16:03:19 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.263225. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46904 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96756 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70045 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75491 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09790 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14685 clock time +[SCHEDULER ITER6 LR1]: 640090 words processed Wed Dec 23 16:03:35 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.262238. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46841 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96107 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68431 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74926 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09617 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14615 clock time +[SCHEDULER ITER6 LR1]: 680075 words processed Wed Dec 23 16:03:51 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.262704. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46743 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96073 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68338 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74869 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09699 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14662 clock time +[SCHEDULER ITER6 LR1]: 720043 words processed Wed Dec 23 16:04:07 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.262781. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47022 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98757 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73504 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76071 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09984 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14776 clock time +[SCHEDULER ITER6 LR1]: 760012 words processed Wed Dec 23 16:04:23 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.262536. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96057 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68614 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74970 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09657 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14671 clock time +[SCHEDULER ITER6 LR1]: 800113 words processed Wed Dec 23 16:04:39 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.262206. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47008 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98136 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10509 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14678 clock time +[SCHEDULER ITER6 LR1]: 840089 words processed Wed Dec 23 16:04:55 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.261061. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46803 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96148 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69172 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75211 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09811 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14737 clock time +[SCHEDULER ITER6 LR1]: 880052 words processed Wed Dec 23 16:05:11 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.260586. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46853 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96314 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69822 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75416 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09877 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14726 clock time [LOG]LMSeqReader: file expires, closing. +(16:05:14 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER6 LR1]: Displaying result: -[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 203.44638459567> <PPL_OOV 180.48040863984> <LOGP -2097552.5682739> -[SCHEDULER ITER6 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 205.48522091656> <PPL_OOV 182.22564154491> <LOGP -2101437.7132322> +[SCHEDULER ITER6 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER6 LR1]: shuffling training file ===PEEK ON TEST 6=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER6 LR1]: 40087 words processed Wed Nov 18 10:37:36 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.270756. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46874 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75384 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.44684 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11462 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:05:16 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:05:16 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:05:16 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:05:16 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:05:16 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:05:16 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:05:16 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER6 LR1]: 40087 words processed Wed Dec 23 16:05:24 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.261357. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49092 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.13956 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.88983 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14360 clock time [LOG]LMSeqReader: file expires, closing. +(16:05:32 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER6 LR1]: Displaying result: -[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 213.76177164248> <PPL_OOV 187.55040025392> <LOGP -187373.11635436> +[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 209.65130071598> <PPL_OOV 182.83339778835> <LOGP -186461.2373622> [SCHEDULER ITER6 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 6=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER6 LR1]: 40095 words processed Wed Nov 18 10:37:45 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.327893. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46740 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74780 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.43821 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11425 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:05:32 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:05:32 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:05:32 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:05:32 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:05:32 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:05:32 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:05:32 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER6 LR1]: 40095 words processed Wed Dec 23 16:05:40 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.318556. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49090 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.18605 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.93529 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14216 clock time [LOG]LMSeqReader: file expires, closing. +(16:05:46 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER6 LR1]: Displaying result: -[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 218.09202962517> <PPL_OOV 196.16969102515> <LOGP -169104.52953794> +[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 214.7899870592> <PPL_OOV 192.22271668783> <LOGP -168453.43547242> [SCHEDULER ITER6 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER6 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.6... +[SCHEDULER ITER6 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.6... [SCHEDULER ITER7 LR1]: preparing parameters... -[SCHEDULER ITER7 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.6... +[SCHEDULER ITER7 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.6... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34539097 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3669 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34542754 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120065 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69128153 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1194416 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69244367 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35796118 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70319220 +reading chunk 5 from 70339858 [SCHEDULER ITER7 LR1]: preparing parameters end. [SCHEDULER ITER7 LR1]: preparing layers... -(10:37:54 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:37:54 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:37:54 2015-11-18)[nerv] info: create layer: combinerL1 -(10:37:54 2015-11-18)[nerv] info: create layer: outputL -(10:37:54 2015-11-18)[nerv] info: create layer: softmaxL -(10:37:54 2015-11-18)[nerv] info: create layer: selectL1 +(16:05:52 2015-12-23)[nerv] info: create layer: sigmoidL1 +(16:05:52 2015-12-23)[nerv] info: create layer: combinerL1 +(16:05:52 2015-12-23)[nerv] info: create layer: selectL1 +(16:05:52 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(16:05:52 2015-12-23)[nerv] info: create layer: outputL +(16:05:52 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(16:05:52 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(16:05:52 2015-12-23)[nerv] info: create layer: softmaxL +(16:05:52 2015-12-23)[nerv] info: create layer: recurrentL1 +(16:05:52 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(16:05:52 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER7 LR1]: preparing layers end. [SCHEDULER ITER7 LR1]: Generate and initing TNN ... +(16:05:52 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -1714,253 +1941,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(16:05:52 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:05:52 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:05:52 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:05:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:05:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:05:52 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER7 LR1]: Initing TNN end. ===ITERATION 7 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER7 LR1]: 40092 words processed Wed Nov 18 10:38:05 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.251501. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47270 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76764 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.31940 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59447 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11461 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11910 clock time -[SCHEDULER ITER7 LR1]: 80099 words processed Wed Nov 18 10:38:17 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.249180. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46812 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73223 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21086 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55670 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09429 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11759 clock time -[SCHEDULER ITER7 LR1]: 120004 words processed Wed Nov 18 10:38:29 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.243853. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46542 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73284 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21655 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55817 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09571 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11723 clock time -[SCHEDULER ITER7 LR1]: 160114 words processed Wed Nov 18 10:38:41 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.246795. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46712 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74229 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25193 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57646 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10261 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11770 clock time -[SCHEDULER ITER7 LR1]: 200066 words processed Wed Nov 18 10:38:53 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.246943. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46211 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73341 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21312 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55895 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09669 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11752 clock time -[SCHEDULER ITER7 LR1]: 240045 words processed Wed Nov 18 10:39:05 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.243492. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46300 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73918 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22383 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56107 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09756 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11811 clock time -[SCHEDULER ITER7 LR1]: 280057 words processed Wed Nov 18 10:39:17 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.240411. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46496 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74737 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26065 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10464 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11824 clock time -[SCHEDULER ITER7 LR1]: 320106 words processed Wed Nov 18 10:39:29 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.237547. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46336 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75410 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26265 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57445 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10285 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11918 clock time -[SCHEDULER ITER7 LR1]: 360024 words processed Wed Nov 18 10:39:41 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.236674. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45882 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74675 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22815 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56278 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09295 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11868 clock time -[SCHEDULER ITER7 LR1]: 400089 words processed Wed Nov 18 10:39:53 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.235629. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46360 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76278 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.28313 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58091 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10666 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11979 clock time -[SCHEDULER ITER7 LR1]: 440067 words processed Wed Nov 18 10:40:05 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.235708. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46147 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74596 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22643 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56116 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09461 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11834 clock time -[SCHEDULER ITER7 LR1]: 480051 words processed Wed Nov 18 10:40:17 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.235789. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45970 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74709 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23079 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56395 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09642 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11909 clock time -[SCHEDULER ITER7 LR1]: 520140 words processed Wed Nov 18 10:40:29 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.235653. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46114 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75427 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26328 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57623 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10376 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11941 clock time -[SCHEDULER ITER7 LR1]: 560132 words processed Wed Nov 18 10:40:41 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.235260. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46032 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74784 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23325 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56328 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09741 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11919 clock time -[SCHEDULER ITER7 LR1]: 600118 words processed Wed Nov 18 10:40:53 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.233722. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46126 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74854 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23290 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56513 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09207 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11975 clock time -[SCHEDULER ITER7 LR1]: 640090 words processed Wed Nov 18 10:41:05 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.232544. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46046 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74810 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23177 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56368 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09477 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11905 clock time -[SCHEDULER ITER7 LR1]: 680075 words processed Wed Nov 18 10:41:17 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.232156. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46161 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75615 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25008 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56772 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09848 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11971 clock time -[SCHEDULER ITER7 LR1]: 720043 words processed Wed Nov 18 10:41:29 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.232032. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45956 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74110 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21620 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55854 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09423 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11793 clock time -[SCHEDULER ITER7 LR1]: 760012 words processed Wed Nov 18 10:41:41 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.232176. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45928 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74325 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22199 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.55926 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09584 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11866 clock time -[SCHEDULER ITER7 LR1]: 800113 words processed Wed Nov 18 10:41:53 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.232236. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45960 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74990 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24589 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57133 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10051 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11802 clock time -[SCHEDULER ITER7 LR1]: 840089 words processed Wed Nov 18 10:42:05 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.231563. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46117 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74825 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23400 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56396 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09560 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11919 clock time -[SCHEDULER ITER7 LR1]: 880052 words processed Wed Nov 18 10:42:17 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.231495. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45982 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74912 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23623 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56339 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09719 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11916 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:05:52 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:05:52 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:05:52 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:05:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:05:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:05:52 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:05:52 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER7 LR1]: 40092 words processed Wed Dec 23 16:06:08 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.245215. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48313 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99556 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.80046 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78645 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11508 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14793 clock time +[SCHEDULER ITER7 LR1]: 80099 words processed Wed Dec 23 16:06:24 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.245071. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48048 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95080 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69536 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74974 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10138 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14703 clock time +[SCHEDULER ITER7 LR1]: 120004 words processed Wed Dec 23 16:06:40 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.241515. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47848 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95124 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69770 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75024 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09981 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14706 clock time +[SCHEDULER ITER7 LR1]: 160114 words processed Wed Dec 23 16:06:56 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.243270. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47897 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97261 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74439 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77059 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10630 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14740 clock time +[SCHEDULER ITER7 LR1]: 200066 words processed Wed Dec 23 16:07:12 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.244089. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47519 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95343 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75184 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09932 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time +[SCHEDULER ITER7 LR1]: 240045 words processed Wed Dec 23 16:07:28 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.239813. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48218 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.01466 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.80679 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77686 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10862 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15003 clock time +[SCHEDULER ITER7 LR1]: 280057 words processed Wed Dec 23 16:07:44 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.237051. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47659 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97221 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74612 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77021 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10737 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14781 clock time +[SCHEDULER ITER7 LR1]: 320106 words processed Wed Dec 23 16:08:00 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.234385. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47415 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97553 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74935 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77212 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10710 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14831 clock time +[SCHEDULER ITER7 LR1]: 360024 words processed Wed Dec 23 16:08:16 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.233491. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47534 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99702 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.77278 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77004 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10552 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14993 clock time +[SCHEDULER ITER7 LR1]: 400089 words processed Wed Dec 23 16:08:32 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.232579. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47195 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97480 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74512 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77353 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10564 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14863 clock time +[SCHEDULER ITER7 LR1]: 440067 words processed Wed Dec 23 16:08:48 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.232677. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47166 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97041 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71836 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75838 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10049 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14838 clock time +[SCHEDULER ITER7 LR1]: 480051 words processed Wed Dec 23 16:09:04 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.232607. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47080 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95623 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69146 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75124 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09882 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14698 clock time +[SCHEDULER ITER7 LR1]: 520140 words processed Wed Dec 23 16:09:20 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.232529. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47166 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97243 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73755 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77026 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10516 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14711 clock time +[SCHEDULER ITER7 LR1]: 560132 words processed Wed Dec 23 16:09:36 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.231957. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46997 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95242 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68296 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75062 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09654 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14687 clock time +[SCHEDULER ITER7 LR1]: 600118 words processed Wed Dec 23 16:09:52 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.233111. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47030 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95610 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68876 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75139 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09653 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14682 clock time +[SCHEDULER ITER7 LR1]: 640090 words processed Wed Dec 23 16:10:08 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.232373. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47007 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95441 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68942 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75272 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09725 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14727 clock time +[SCHEDULER ITER7 LR1]: 680075 words processed Wed Dec 23 16:10:24 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.232275. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47039 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95947 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69623 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09788 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14724 clock time +[SCHEDULER ITER7 LR1]: 720043 words processed Wed Dec 23 16:10:40 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.232260. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47017 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95948 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70285 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75604 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09991 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14745 clock time +[SCHEDULER ITER7 LR1]: 760012 words processed Wed Dec 23 16:10:56 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.232134. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95396 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68561 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75106 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09628 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14651 clock time +[SCHEDULER ITER7 LR1]: 800113 words processed Wed Dec 23 16:11:12 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.232050. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47145 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97652 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74708 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77437 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10535 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14822 clock time +[SCHEDULER ITER7 LR1]: 840089 words processed Wed Dec 23 16:11:28 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.230891. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46908 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95352 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68471 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75039 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09765 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14704 clock time +[SCHEDULER ITER7 LR1]: 880052 words processed Wed Dec 23 16:11:44 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.230637. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47013 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95335 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69001 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75211 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09783 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14692 clock time [LOG]LMSeqReader: file expires, closing. +(16:11:47 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER7 LR1]: Displaying result: -[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 191.66281054712> <PPL_OOV 170.45050083613> <LOGP -2074469.2150481> -[SCHEDULER ITER7 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 191.23533642528> <PPL_OOV 170.09507366409> <LOGP -2073626.4998509> +[SCHEDULER ITER7 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER7 LR1]: shuffling training file ===PEEK ON TEST 7=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER7 LR1]: 40087 words processed Wed Nov 18 10:42:26 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.250935. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46872 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75737 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.45120 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11487 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:11:49 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:11:49 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:11:49 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:11:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:11:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:11:49 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:11:49 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER7 LR1]: 40087 words processed Wed Dec 23 16:11:57 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.246495. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49160 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.13914 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.89130 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14356 clock time [LOG]LMSeqReader: file expires, closing. +(16:12:05 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER7 LR1]: Displaying result: -[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 206.26330245608> <PPL_OOV 178.68977420066> <LOGP -185640.57827444> +[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 202.3198797128> <PPL_OOV 176.56637282715> <LOGP -185212.62606652> [SCHEDULER ITER7 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 7=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER7 LR1]: 40095 words processed Wed Nov 18 10:42:35 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.308122. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46668 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75255 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.44319 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11436 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:12:05 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:12:05 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:12:05 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:12:05 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:12:05 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:12:05 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:12:05 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER7 LR1]: 40095 words processed Wed Dec 23 16:12:13 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.305856. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49183 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.18578 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.93690 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14266 clock time [LOG]LMSeqReader: file expires, closing. +(16:12:19 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER7 LR1]: Displaying result: -[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 210.66094197492> <PPL_OOV 187.46229275885> <LOGP -167650.13106111> +[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 207.59243440551> <PPL_OOV 185.94614197065> <LOGP -167389.99780692> [SCHEDULER ITER7 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER7 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.7... +[SCHEDULER ITER7 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.7... [SCHEDULER ITER8 LR1]: preparing parameters... -[SCHEDULER ITER8 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.7... +[SCHEDULER ITER8 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.7... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1074480 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3669 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1078136 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120077 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35669430 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1193722 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35785661 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35803139 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70326762 +reading chunk 5 from 70348369 [SCHEDULER ITER8 LR1]: preparing parameters end. [SCHEDULER ITER8 LR1]: preparing layers... -(10:42:44 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:42:44 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:42:44 2015-11-18)[nerv] info: create layer: combinerL1 -(10:42:44 2015-11-18)[nerv] info: create layer: outputL -(10:42:44 2015-11-18)[nerv] info: create layer: softmaxL -(10:42:44 2015-11-18)[nerv] info: create layer: selectL1 +(16:12:25 2015-12-23)[nerv] info: create layer: sigmoidL1 +(16:12:25 2015-12-23)[nerv] info: create layer: combinerL1 +(16:12:25 2015-12-23)[nerv] info: create layer: selectL1 +(16:12:25 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(16:12:25 2015-12-23)[nerv] info: create layer: outputL +(16:12:25 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(16:12:25 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(16:12:25 2015-12-23)[nerv] info: create layer: softmaxL +(16:12:25 2015-12-23)[nerv] info: create layer: recurrentL1 +(16:12:25 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(16:12:25 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER8 LR1]: preparing layers end. [SCHEDULER ITER8 LR1]: Generate and initing TNN ... +(16:12:25 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -1976,253 +2239,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(16:12:25 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:12:25 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:12:25 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:12:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:12:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:12:25 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER8 LR1]: Initing TNN end. ===ITERATION 8 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER8 LR1]: 40092 words processed Wed Nov 18 10:42:55 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.223485. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47200 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75378 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.29123 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59231 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10712 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11986 clock time -[SCHEDULER ITER8 LR1]: 80099 words processed Wed Nov 18 10:43:07 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.223158. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47187 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74128 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23784 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56720 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09445 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11953 clock time -[SCHEDULER ITER8 LR1]: 120004 words processed Wed Nov 18 10:43:19 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.219440. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47016 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74111 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24157 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56875 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09419 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11947 clock time -[SCHEDULER ITER8 LR1]: 160114 words processed Wed Nov 18 10:43:31 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.222712. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47032 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75221 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27212 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58467 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09944 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11994 clock time -[SCHEDULER ITER8 LR1]: 200066 words processed Wed Nov 18 10:43:43 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.223369. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46715 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74746 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25209 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57510 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09616 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11949 clock time -[SCHEDULER ITER8 LR1]: 240045 words processed Wed Nov 18 10:43:55 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.220367. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46430 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73739 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21838 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56495 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.08943 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11917 clock time -[SCHEDULER ITER8 LR1]: 280057 words processed Wed Nov 18 10:44:07 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.217273. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46779 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75278 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58757 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10111 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11987 clock time -[SCHEDULER ITER8 LR1]: 320106 words processed Wed Nov 18 10:44:19 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.214817. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46468 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74785 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26351 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58130 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10206 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11979 clock time -[SCHEDULER ITER8 LR1]: 360024 words processed Wed Nov 18 10:44:31 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.214075. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45968 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73531 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22015 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56236 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09857 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11860 clock time -[SCHEDULER ITER8 LR1]: 400089 words processed Wed Nov 18 10:44:43 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.213366. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46577 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75979 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.29753 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59223 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11076 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12017 clock time -[SCHEDULER ITER8 LR1]: 440067 words processed Wed Nov 18 10:44:55 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.213651. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46208 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74276 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24185 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10050 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11936 clock time -[SCHEDULER ITER8 LR1]: 480051 words processed Wed Nov 18 10:45:07 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.214115. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46236 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74395 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23556 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57228 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09294 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11950 clock time -[SCHEDULER ITER8 LR1]: 520140 words processed Wed Nov 18 10:45:19 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.214208. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46429 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75450 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27509 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58817 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09921 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11984 clock time -[SCHEDULER ITER8 LR1]: 560132 words processed Wed Nov 18 10:45:31 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.213975. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46060 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74065 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22367 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56833 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09161 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11959 clock time -[SCHEDULER ITER8 LR1]: 600118 words processed Wed Nov 18 10:45:43 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.213825. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46104 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74419 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22681 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56828 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09026 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11865 clock time -[SCHEDULER ITER8 LR1]: 640090 words processed Wed Nov 18 10:45:55 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.212654. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46274 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74627 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24920 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57413 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10174 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11935 clock time -[SCHEDULER ITER8 LR1]: 680075 words processed Wed Nov 18 10:46:07 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.212618. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46054 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74089 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23804 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56904 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10392 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11977 clock time -[SCHEDULER ITER8 LR1]: 720043 words processed Wed Nov 18 10:46:19 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.212644. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46065 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74309 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24161 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57031 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10304 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11893 clock time -[SCHEDULER ITER8 LR1]: 760012 words processed Wed Nov 18 10:46:31 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.212618. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46054 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74347 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22727 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56723 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09189 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11893 clock time -[SCHEDULER ITER8 LR1]: 800113 words processed Wed Nov 18 10:46:43 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.212799. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74704 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26342 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10933 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11905 clock time -[SCHEDULER ITER8 LR1]: 840089 words processed Wed Nov 18 10:46:55 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.211971. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46022 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74098 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23139 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56643 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10115 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11903 clock time -[SCHEDULER ITER8 LR1]: 880052 words processed Wed Nov 18 10:47:07 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.211963. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46174 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74639 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24128 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57056 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09675 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11954 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:12:25 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:12:25 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:12:25 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:12:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:12:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:12:25 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:12:25 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER8 LR1]: 40092 words processed Wed Dec 23 16:12:41 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.222096. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48786 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.03901 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.86863 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.79636 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11996 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15003 clock time +[SCHEDULER ITER8 LR1]: 80099 words processed Wed Dec 23 16:12:57 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.221006. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47947 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94689 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68008 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74236 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10057 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14626 clock time +[SCHEDULER ITER8 LR1]: 120004 words processed Wed Dec 23 16:13:13 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.217219. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48053 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97278 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73095 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75469 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10299 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14834 clock time +[SCHEDULER ITER8 LR1]: 160114 words processed Wed Dec 23 16:13:29 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.219516. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47845 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96947 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73464 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76461 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10742 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14711 clock time +[SCHEDULER ITER8 LR1]: 200066 words processed Wed Dec 23 16:13:45 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.220067. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47374 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94925 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67843 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09869 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14661 clock time +[SCHEDULER ITER8 LR1]: 240045 words processed Wed Dec 23 16:14:01 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.215749. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94766 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67469 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74301 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09872 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14621 clock time +[SCHEDULER ITER8 LR1]: 280057 words processed Wed Dec 23 16:14:17 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.212181. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48016 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.01012 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.81014 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78328 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11239 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14933 clock time +[SCHEDULER ITER8 LR1]: 320106 words processed Wed Dec 23 16:14:33 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.209706. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47329 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96793 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72917 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76435 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10704 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14664 clock time +[SCHEDULER ITER8 LR1]: 360024 words processed Wed Dec 23 16:14:49 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.208773. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46941 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94797 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67371 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74426 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09772 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14602 clock time +[SCHEDULER ITER8 LR1]: 400089 words processed Wed Dec 23 16:15:05 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.207512. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47078 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97215 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76537 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10645 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14677 clock time +[SCHEDULER ITER8 LR1]: 440067 words processed Wed Dec 23 16:15:21 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.207774. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46931 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95051 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67520 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74454 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09863 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14609 clock time +[SCHEDULER ITER8 LR1]: 480051 words processed Wed Dec 23 16:15:37 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.208015. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46936 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94923 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67511 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74602 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09972 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14609 clock time +[SCHEDULER ITER8 LR1]: 520140 words processed Wed Dec 23 16:15:53 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.208008. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47120 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96932 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72834 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76567 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10681 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14662 clock time +[SCHEDULER ITER8 LR1]: 560132 words processed Wed Dec 23 16:16:09 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.207602. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46949 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94982 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67429 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74576 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09836 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14615 clock time +[SCHEDULER ITER8 LR1]: 600118 words processed Wed Dec 23 16:16:25 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.206694. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46919 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94807 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67259 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74389 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09980 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14604 clock time +[SCHEDULER ITER8 LR1]: 640090 words processed Wed Dec 23 16:16:41 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.205599. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46862 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94767 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67107 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74390 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09986 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14589 clock time +[SCHEDULER ITER8 LR1]: 680075 words processed Wed Dec 23 16:16:57 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.206595. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46874 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94865 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67122 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74330 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09877 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14647 clock time +[SCHEDULER ITER8 LR1]: 720043 words processed Wed Dec 23 16:17:13 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.207275. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46855 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94964 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67661 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74529 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09969 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14648 clock time +[SCHEDULER ITER8 LR1]: 760012 words processed Wed Dec 23 16:17:29 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.208178. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46843 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94842 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67289 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74296 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10036 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14590 clock time +[SCHEDULER ITER8 LR1]: 800113 words processed Wed Dec 23 16:17:45 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.208513. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47007 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97018 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72470 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10575 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14656 clock time +[SCHEDULER ITER8 LR1]: 840089 words processed Wed Dec 23 16:18:01 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.207768. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46866 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94900 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67188 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74342 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09897 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14628 clock time +[SCHEDULER ITER8 LR1]: 880052 words processed Wed Dec 23 16:18:17 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.207814. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47480 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00089 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.77654 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77100 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10734 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14957 clock time [LOG]LMSeqReader: file expires, closing. +(16:18:20 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER8 LR1]: Displaying result: -[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 182.85875939578> <PPL_OOV 162.9490849551> <LOGP -2056299.1378532> -[SCHEDULER ITER8 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 181.08140162668> <PPL_OOV 161.40779308922> <LOGP -2052462.3275596> +[SCHEDULER ITER8 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER8 LR1]: shuffling training file ===PEEK ON TEST 8=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER8 LR1]: 40087 words processed Wed Nov 18 10:47:16 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.242961. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47194 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75875 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.45598 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11525 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:18:22 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:18:22 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:18:22 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:18:22 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:18:22 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:18:22 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:18:22 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER8 LR1]: 40087 words processed Wed Dec 23 16:18:30 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.234777. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49067 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12408 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.87451 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14271 clock time [LOG]LMSeqReader: file expires, closing. +(16:18:38 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER8 LR1]: Displaying result: -[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 200.73582668906> <PPL_OOV 175.37028924635> <LOGP -184969.29458693> +[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 195.92634498016> <PPL_OOV 171.97290570376> <LOGP -184268.97021437> [SCHEDULER ITER8 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 8=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER8 LR1]: 40095 words processed Wed Nov 18 10:47:25 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.301838. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46955 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75135 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.44506 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11486 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:18:38 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:18:38 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:18:38 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:18:38 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:18:38 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:18:38 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:18:38 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER8 LR1]: 40095 words processed Wed Dec 23 16:18:46 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.295830. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49146 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17239 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92280 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14213 clock time [LOG]LMSeqReader: file expires, closing. +(16:18:52 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER8 LR1]: Displaying result: -[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 204.97925383417> <PPL_OOV 183.63222465748> <LOGP -166988.87054074> +[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 201.40202999686> <PPL_OOV 181.14831866321> <LOGP -166552.61051856> [SCHEDULER ITER8 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER8 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.8... +[SCHEDULER ITER8 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.8... [SCHEDULER ITER9 LR1]: preparing parameters... -[SCHEDULER ITER9 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.8... +[SCHEDULER ITER9 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.8... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34542686 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3668 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34546343 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120063 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69143526 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1193064 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69259775 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35809896 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70333851 +reading chunk 5 from 70356242 [SCHEDULER ITER9 LR1]: preparing parameters end. [SCHEDULER ITER9 LR1]: preparing layers... -(10:47:34 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:47:34 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:47:34 2015-11-18)[nerv] info: create layer: combinerL1 -(10:47:34 2015-11-18)[nerv] info: create layer: outputL -(10:47:34 2015-11-18)[nerv] info: create layer: softmaxL -(10:47:34 2015-11-18)[nerv] info: create layer: selectL1 +(16:18:58 2015-12-23)[nerv] info: create layer: sigmoidL1 +(16:18:58 2015-12-23)[nerv] info: create layer: combinerL1 +(16:18:58 2015-12-23)[nerv] info: create layer: selectL1 +(16:18:58 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(16:18:58 2015-12-23)[nerv] info: create layer: outputL +(16:18:58 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(16:18:58 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(16:18:58 2015-12-23)[nerv] info: create layer: softmaxL +(16:18:58 2015-12-23)[nerv] info: create layer: recurrentL1 +(16:18:58 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(16:18:58 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER9 LR1]: preparing layers end. [SCHEDULER ITER9 LR1]: Generate and initing TNN ... +(16:18:58 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -2238,253 +2537,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(16:18:58 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:18:58 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:18:58 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:18:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:18:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:18:58 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER9 LR1]: Initing TNN end. ===ITERATION 9 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER9 LR1]: 40092 words processed Wed Nov 18 10:47:45 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.206042. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47050 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74971 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.28213 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58715 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10997 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11997 clock time -[SCHEDULER ITER9 LR1]: 80099 words processed Wed Nov 18 10:47:57 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.206343. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47213 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74031 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25120 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57189 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10262 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12019 clock time -[SCHEDULER ITER9 LR1]: 120004 words processed Wed Nov 18 10:48:09 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.202072. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47115 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74334 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24743 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56756 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09868 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11981 clock time -[SCHEDULER ITER9 LR1]: 160114 words processed Wed Nov 18 10:48:21 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.206597. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46962 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74532 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26954 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58197 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10828 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12014 clock time -[SCHEDULER ITER9 LR1]: 200066 words processed Wed Nov 18 10:48:33 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.206957. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46621 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73933 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23991 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56863 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:18:58 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:18:58 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:18:58 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:18:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:18:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:18:58 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:18:58 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER9 LR1]: 40092 words processed Wed Dec 23 16:19:14 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.209599. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48168 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97518 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76522 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77292 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11611 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14694 clock time +[SCHEDULER ITER9 LR1]: 80099 words processed Wed Dec 23 16:19:30 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.205604. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48036 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94159 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67938 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74209 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10220 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14604 clock time +[SCHEDULER ITER9 LR1]: 120004 words processed Wed Dec 23 16:19:46 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.199816. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94153 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74196 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09984 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14622 clock time +[SCHEDULER ITER9 LR1]: 160114 words processed Wed Dec 23 16:20:02 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.201050. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47941 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96841 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74040 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76752 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10748 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14742 clock time +[SCHEDULER ITER9 LR1]: 200066 words processed Wed Dec 23 16:20:18 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.201623. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47379 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94262 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67500 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74287 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10075 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12020 clock time -[SCHEDULER ITER9 LR1]: 240045 words processed Wed Nov 18 10:48:45 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.202669. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46622 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74566 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25139 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56860 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10550 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12000 clock time -[SCHEDULER ITER9 LR1]: 280057 words processed Wed Nov 18 10:48:57 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.199498. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46745 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74662 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26429 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58046 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10272 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11980 clock time -[SCHEDULER ITER9 LR1]: 320106 words processed Wed Nov 18 10:49:09 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.197205. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46554 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74500 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26464 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57759 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11096 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11959 clock time -[SCHEDULER ITER9 LR1]: 360024 words processed Wed Nov 18 10:49:21 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.196210. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45992 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73450 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22412 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56114 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10498 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11900 clock time -[SCHEDULER ITER9 LR1]: 400089 words processed Wed Nov 18 10:49:33 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.195749. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46254 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74420 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25369 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57755 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10525 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11970 clock time -[SCHEDULER ITER9 LR1]: 440067 words processed Wed Nov 18 10:49:45 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.196194. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46419 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74687 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25038 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57106 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10356 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11995 clock time -[SCHEDULER ITER9 LR1]: 480051 words processed Wed Nov 18 10:49:57 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.196723. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46786 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76705 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.29484 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58496 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10636 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12151 clock time -[SCHEDULER ITER9 LR1]: 520140 words processed Wed Nov 18 10:50:09 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.196745. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46525 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75277 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27749 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58528 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10674 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12067 clock time -[SCHEDULER ITER9 LR1]: 560132 words processed Wed Nov 18 10:50:21 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.196443. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46284 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73739 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24005 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56972 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10715 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12050 clock time -[SCHEDULER ITER9 LR1]: 600118 words processed Wed Nov 18 10:50:33 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.195960. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46220 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73731 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22816 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56571 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10159 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11897 clock time -[SCHEDULER ITER9 LR1]: 640090 words processed Wed Nov 18 10:50:45 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.194630. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46790 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76519 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.30552 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58990 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11086 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12190 clock time -[SCHEDULER ITER9 LR1]: 680075 words processed Wed Nov 18 10:50:57 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.194485. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46120 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72829 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21634 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56679 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09896 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11895 clock time -[SCHEDULER ITER9 LR1]: 720043 words processed Wed Nov 18 10:51:09 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.194576. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45990 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73048 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56110 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09525 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11886 clock time -[SCHEDULER ITER9 LR1]: 760012 words processed Wed Nov 18 10:51:21 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.194645. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46073 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73109 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21916 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56221 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10346 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11862 clock time -[SCHEDULER ITER9 LR1]: 800113 words processed Wed Nov 18 10:51:33 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.194792. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46146 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73565 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57883 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10019 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11928 clock time -[SCHEDULER ITER9 LR1]: 840089 words processed Wed Nov 18 10:51:45 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.193957. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46098 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73214 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.21389 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56082 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10031 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11843 clock time -[SCHEDULER ITER9 LR1]: 880052 words processed Wed Nov 18 10:51:57 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.194063. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45976 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72775 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20289 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56160 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09285 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11833 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14606 clock time +[SCHEDULER ITER9 LR1]: 240045 words processed Wed Dec 23 16:20:34 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.197338. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47409 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94114 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67271 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74239 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10120 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14602 clock time +[SCHEDULER ITER9 LR1]: 280057 words processed Wed Dec 23 16:20:50 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.193627. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47530 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96260 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72697 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76290 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10709 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14658 clock time +[SCHEDULER ITER9 LR1]: 320106 words processed Wed Dec 23 16:21:06 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.191211. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47359 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96500 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72863 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76475 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10590 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14704 clock time +[SCHEDULER ITER9 LR1]: 360024 words processed Wed Dec 23 16:21:22 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.190157. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94062 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74302 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09936 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14637 clock time +[SCHEDULER ITER9 LR1]: 400089 words processed Wed Dec 23 16:21:38 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.190321. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47115 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96228 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72159 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76301 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10812 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14661 clock time +[SCHEDULER ITER9 LR1]: 440067 words processed Wed Dec 23 16:21:54 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.190628. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46910 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94062 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66524 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74261 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09871 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14627 clock time +[SCHEDULER ITER9 LR1]: 480051 words processed Wed Dec 23 16:22:10 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.190725. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46960 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94514 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67235 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74463 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09985 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14637 clock time +[SCHEDULER ITER9 LR1]: 520140 words processed Wed Dec 23 16:22:26 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.190558. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47082 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96219 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72125 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10718 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14649 clock time +[SCHEDULER ITER9 LR1]: 560132 words processed Wed Dec 23 16:22:42 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.190084. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46913 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94087 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66515 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74306 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09970 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14594 clock time +[SCHEDULER ITER9 LR1]: 600118 words processed Wed Dec 23 16:22:58 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.188470. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46871 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94150 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66627 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74351 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09817 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14653 clock time +[SCHEDULER ITER9 LR1]: 640090 words processed Wed Dec 23 16:23:14 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.187666. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47147 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96204 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71072 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75715 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10303 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time +[SCHEDULER ITER9 LR1]: 680075 words processed Wed Dec 23 16:23:30 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.186989. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47024 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96401 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71100 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75543 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10388 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14786 clock time +[SCHEDULER ITER9 LR1]: 720043 words processed Wed Dec 23 16:23:46 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.186968. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46971 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94149 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67311 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74540 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10065 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14663 clock time +[SCHEDULER ITER9 LR1]: 760012 words processed Wed Dec 23 16:24:02 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.187343. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46899 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94551 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67673 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74637 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09936 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14663 clock time +[SCHEDULER ITER9 LR1]: 800113 words processed Wed Dec 23 16:24:18 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.187542. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47383 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98123 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76763 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77700 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11400 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14838 clock time +[SCHEDULER ITER9 LR1]: 840089 words processed Wed Dec 23 16:24:34 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.186627. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46844 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94014 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66629 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74379 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10046 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14594 clock time +[SCHEDULER ITER9 LR1]: 880052 words processed Wed Dec 23 16:24:50 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.186624. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46911 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94153 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66944 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74363 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10020 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14605 clock time [LOG]LMSeqReader: file expires, closing. +(16:24:53 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER9 LR1]: Displaying result: -[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 175.15529644178> <PPL_OOV 156.3706229079> <LOGP -2039662.5067889> -[SCHEDULER ITER9 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 172.04697427386> <PPL_OOV 153.71798264245> <LOGP -2032755.2050744> +[SCHEDULER ITER9 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER9 LR1]: shuffling training file ===PEEK ON TEST 9=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER9 LR1]: 40087 words processed Wed Nov 18 10:52:06 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.232448. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47015 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75902 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.45569 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11574 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:24:55 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:24:55 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:24:55 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:24:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:24:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:24:55 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:24:55 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER9 LR1]: 40087 words processed Wed Dec 23 16:25:03 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.222673. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49127 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.13177 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.88385 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14315 clock time [LOG]LMSeqReader: file expires, closing. +(16:25:11 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER9 LR1]: Displaying result: -[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 196.68771340668> <PPL_OOV 171.28145024784> <LOGP -184124.74262498> +[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 190.71276732655> <PPL_OOV 167.21121573361> <LOGP -183263.76729209> [SCHEDULER ITER9 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 9=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER9 LR1]: 40095 words processed Wed Nov 18 10:52:16 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.292322. - [global_conf.timer]: time spent on tnn_beforeprocess:0.50580 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89009 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.64386 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12357 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:25:11 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:25:11 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:25:11 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:25:11 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:25:11 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:25:11 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:25:11 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER9 LR1]: 40095 words processed Wed Dec 23 16:25:19 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.283727. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49255 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17887 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.93193 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14250 clock time [LOG]LMSeqReader: file expires, closing. +(16:25:25 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER9 LR1]: Displaying result: -[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 200.74808824921> <PPL_OOV 179.37500626499> <LOGP -166237.47990123> +[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 195.98088075761> <PPL_OOV 176.13032605899> <LOGP -165652.72609855> [SCHEDULER ITER9 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER9 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.9... +[SCHEDULER ITER9 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.9... [SCHEDULER ITER10 LR1]: preparing parameters... -[SCHEDULER ITER10 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.9... +[SCHEDULER ITER10 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.9... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1073504 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3668 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1077161 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120072 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35679936 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1192417 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35796199 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35816192 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70340150 +reading chunk 5 from 70363379 [SCHEDULER ITER10 LR1]: preparing parameters end. [SCHEDULER ITER10 LR1]: preparing layers... -(10:52:26 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:52:26 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:52:26 2015-11-18)[nerv] info: create layer: combinerL1 -(10:52:26 2015-11-18)[nerv] info: create layer: outputL -(10:52:26 2015-11-18)[nerv] info: create layer: softmaxL -(10:52:26 2015-11-18)[nerv] info: create layer: selectL1 +(16:25:31 2015-12-23)[nerv] info: create layer: sigmoidL1 +(16:25:31 2015-12-23)[nerv] info: create layer: combinerL1 +(16:25:31 2015-12-23)[nerv] info: create layer: selectL1 +(16:25:31 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(16:25:31 2015-12-23)[nerv] info: create layer: outputL +(16:25:31 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(16:25:31 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(16:25:31 2015-12-23)[nerv] info: create layer: softmaxL +(16:25:31 2015-12-23)[nerv] info: create layer: recurrentL1 +(16:25:31 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(16:25:31 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER10 LR1]: preparing layers end. [SCHEDULER ITER10 LR1]: Generate and initing TNN ... +(16:25:31 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -2500,253 +2835,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(16:25:31 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:25:31 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:25:31 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:25:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:25:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:25:31 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER10 LR1]: Initing TNN end. ===ITERATION 10 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER10 LR1]: 40092 words processed Wed Nov 18 10:52:37 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.187626. - [global_conf.timer]: time spent on tnn_beforeprocess:0.49034 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.83385 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.53127 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66879 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13693 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12625 clock time -[SCHEDULER ITER10 LR1]: 80099 words processed Wed Nov 18 10:52:49 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.187185. - [global_conf.timer]: time spent on tnn_beforeprocess:0.51138 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94141 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.79346 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.73667 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.16559 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13340 clock time -[SCHEDULER ITER10 LR1]: 120004 words processed Wed Nov 18 10:53:01 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.183663. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47670 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.78031 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.34401 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59273 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11058 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12225 clock time -[SCHEDULER ITER10 LR1]: 160114 words processed Wed Nov 18 10:53:13 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.186533. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47405 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76657 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.31822 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59381 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10987 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12139 clock time -[SCHEDULER ITER10 LR1]: 200066 words processed Wed Nov 18 10:53:25 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.187822. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46812 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74876 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26041 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57316 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10123 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12045 clock time -[SCHEDULER ITER10 LR1]: 240045 words processed Wed Nov 18 10:53:37 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.184261. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46663 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75239 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26592 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57550 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10219 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12015 clock time -[SCHEDULER ITER10 LR1]: 280057 words processed Wed Nov 18 10:53:49 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.181682. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46951 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76315 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.30757 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59070 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11089 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11997 clock time -[SCHEDULER ITER10 LR1]: 320106 words processed Wed Nov 18 10:54:01 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.179709. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48108 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.82285 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48257 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64562 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13202 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12704 clock time -[SCHEDULER ITER10 LR1]: 360024 words processed Wed Nov 18 10:54:13 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.178820. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46470 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74475 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24827 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56957 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09895 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11965 clock time -[SCHEDULER ITER10 LR1]: 400089 words processed Wed Nov 18 10:54:25 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.177844. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46510 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75564 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27926 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58299 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10635 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12006 clock time -[SCHEDULER ITER10 LR1]: 440067 words processed Wed Nov 18 10:54:37 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.178455. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46370 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74711 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24594 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56957 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09929 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11961 clock time -[SCHEDULER ITER10 LR1]: 480051 words processed Wed Nov 18 10:54:49 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.178986. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46607 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75624 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27252 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57857 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10255 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12072 clock time -[SCHEDULER ITER10 LR1]: 520140 words processed Wed Nov 18 10:55:01 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.179147. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46629 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76321 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.30277 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59194 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10976 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12013 clock time -[SCHEDULER ITER10 LR1]: 560132 words processed Wed Nov 18 10:55:13 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.179057. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46199 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74421 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22938 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56597 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09304 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11943 clock time -[SCHEDULER ITER10 LR1]: 600118 words processed Wed Nov 18 10:55:25 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.178715. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46295 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74415 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23768 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56694 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09883 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11922 clock time -[SCHEDULER ITER10 LR1]: 640090 words processed Wed Nov 18 10:55:37 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.177619. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46282 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74450 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23064 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56633 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09281 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11885 clock time -[SCHEDULER ITER10 LR1]: 680075 words processed Wed Nov 18 10:55:49 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.177353. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46317 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74162 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22982 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56488 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09711 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11862 clock time -[SCHEDULER ITER10 LR1]: 720043 words processed Wed Nov 18 10:56:01 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.177437. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46259 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74441 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23905 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56734 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09788 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11947 clock time -[SCHEDULER ITER10 LR1]: 760012 words processed Wed Nov 18 10:56:13 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.177489. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46279 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74377 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56696 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09629 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11940 clock time -[SCHEDULER ITER10 LR1]: 800113 words processed Wed Nov 18 10:56:25 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.177808. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46449 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75256 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27073 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58336 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10163 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11982 clock time -[SCHEDULER ITER10 LR1]: 840089 words processed Wed Nov 18 10:56:37 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.177156. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46207 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74318 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23297 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56643 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09597 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11963 clock time -[SCHEDULER ITER10 LR1]: 880052 words processed Wed Nov 18 10:56:49 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.177345. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46358 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74369 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23957 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56683 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09873 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11930 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:25:31 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:25:31 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:25:31 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:25:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:25:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:25:31 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:25:31 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER10 LR1]: 40092 words processed Wed Dec 23 16:25:47 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.200838. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47966 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96725 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74566 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76981 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11308 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14694 clock time +[SCHEDULER ITER10 LR1]: 80099 words processed Wed Dec 23 16:26:03 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.191865. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47979 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94662 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68360 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74408 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10072 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14729 clock time +[SCHEDULER ITER10 LR1]: 120004 words processed Wed Dec 23 16:26:19 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.183583. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48952 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.04215 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.89767 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.79992 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11782 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15693 clock time +[SCHEDULER ITER10 LR1]: 160114 words processed Wed Dec 23 16:26:35 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.184121. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48899 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.04053 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.88592 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.80459 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11928 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15226 clock time +[SCHEDULER ITER10 LR1]: 200066 words processed Wed Dec 23 16:26:51 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.184003. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47318 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93077 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66547 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74612 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09921 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14680 clock time +[SCHEDULER ITER10 LR1]: 240045 words processed Wed Dec 23 16:27:07 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.179592. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47454 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93199 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66448 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74452 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09795 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14699 clock time +[SCHEDULER ITER10 LR1]: 280057 words processed Wed Dec 23 16:27:23 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.175536. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95279 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71710 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76228 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10652 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14742 clock time +[SCHEDULER ITER10 LR1]: 320106 words processed Wed Dec 23 16:27:39 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.173063. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47316 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95205 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71035 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76130 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10415 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14738 clock time +[SCHEDULER ITER10 LR1]: 360024 words processed Wed Dec 23 16:27:55 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.171861. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93121 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65589 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74152 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09568 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14707 clock time +[SCHEDULER ITER10 LR1]: 400089 words processed Wed Dec 23 16:28:11 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.170365. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47249 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96079 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72494 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76551 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10591 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14841 clock time +[SCHEDULER ITER10 LR1]: 440067 words processed Wed Dec 23 16:28:27 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.170815. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47196 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95362 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69596 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75057 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10132 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14826 clock time +[SCHEDULER ITER10 LR1]: 480051 words processed Wed Dec 23 16:28:43 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.171191. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46936 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93100 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65316 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74138 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09717 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time +[SCHEDULER ITER10 LR1]: 520140 words processed Wed Dec 23 16:28:59 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.171141. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47082 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95221 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70659 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76134 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10378 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14738 clock time +[SCHEDULER ITER10 LR1]: 560132 words processed Wed Dec 23 16:29:15 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.170811. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47036 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93629 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74290 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09649 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14732 clock time +[SCHEDULER ITER10 LR1]: 600118 words processed Wed Dec 23 16:29:31 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.169298. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46930 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93174 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65310 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74047 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09719 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14689 clock time +[SCHEDULER ITER10 LR1]: 640090 words processed Wed Dec 23 16:29:47 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.168092. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46886 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93264 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65823 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74300 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09910 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14692 clock time +[SCHEDULER ITER10 LR1]: 680075 words processed Wed Dec 23 16:30:03 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.167670. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46917 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93684 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74190 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09759 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14673 clock time +[SCHEDULER ITER10 LR1]: 720043 words processed Wed Dec 23 16:30:19 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.167781. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47001 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93747 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67617 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74970 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10136 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14728 clock time +[SCHEDULER ITER10 LR1]: 760012 words processed Wed Dec 23 16:30:35 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.168953. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46877 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93182 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65525 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74115 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09786 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14667 clock time +[SCHEDULER ITER10 LR1]: 800113 words processed Wed Dec 23 16:30:51 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.169778. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47074 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95334 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70963 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76216 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10496 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14754 clock time +[SCHEDULER ITER10 LR1]: 840089 words processed Wed Dec 23 16:31:07 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.169210. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46900 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93153 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65576 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74247 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09767 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14746 clock time +[SCHEDULER ITER10 LR1]: 880052 words processed Wed Dec 23 16:31:23 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.169470. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46875 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93202 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65904 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74309 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09867 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14704 clock time [LOG]LMSeqReader: file expires, closing. +(16:31:26 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER10 LR1]: Displaying result: -[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 168.28451804124> <PPL_OOV 150.46390246938> <LOGP -2024117.1344033> -[SCHEDULER ITER10 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 165.13314282506> <PPL_OOV 147.78027341883> <LOGP -2016851.6007733> +[SCHEDULER ITER10 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER10 LR1]: shuffling training file ===PEEK ON TEST 10=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER10 LR1]: 40087 words processed Wed Nov 18 10:56:58 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.226332. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76387 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.46298 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11545 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:31:28 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:31:28 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:31:28 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:31:28 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:31:28 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:31:28 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:31:28 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER10 LR1]: 40087 words processed Wed Dec 23 16:31:36 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.216476. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49159 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12114 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.87563 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14402 clock time [LOG]LMSeqReader: file expires, closing. +(16:31:44 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER10 LR1]: Displaying result: -[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 193.84009884531> <PPL_OOV 168.90002809503> <LOGP -183623.51796897> +[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 188.12627709733> <PPL_OOV 164.49058152389> <LOGP -182676.50559485> [SCHEDULER ITER10 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 10=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER10 LR1]: 40095 words processed Wed Nov 18 10:57:07 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.288461. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47282 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75930 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.45694 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11482 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:31:44 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:31:44 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:31:44 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:31:44 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:31:44 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:31:44 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:31:44 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER10 LR1]: 40095 words processed Wed Dec 23 16:31:52 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.278231. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49407 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17411 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.93076 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14407 clock time [LOG]LMSeqReader: file expires, closing. +(16:31:58 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER10 LR1]: Displaying result: -[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 198.49192596558> <PPL_OOV 177.42026715645> <LOGP -165886.47816098> +[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 193.70824489988> <PPL_OOV 173.68462319488> <LOGP -165204.79870982> [SCHEDULER ITER10 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER10 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.10... +[SCHEDULER ITER10 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.10... [SCHEDULER ITER11 LR1]: preparing parameters... -[SCHEDULER ITER11 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.10... +[SCHEDULER ITER11 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.10... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34544696 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3667 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34548353 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120065 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69157059 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1191872 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69273324 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35823208 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70346338 +reading chunk 5 from 70371029 [SCHEDULER ITER11 LR1]: preparing parameters end. [SCHEDULER ITER11 LR1]: preparing layers... -(10:57:16 2015-11-18)[nerv] info: create layer: recurrentL1 -(10:57:16 2015-11-18)[nerv] info: create layer: sigmoidL1 -(10:57:16 2015-11-18)[nerv] info: create layer: combinerL1 -(10:57:16 2015-11-18)[nerv] info: create layer: outputL -(10:57:16 2015-11-18)[nerv] info: create layer: softmaxL -(10:57:16 2015-11-18)[nerv] info: create layer: selectL1 +(16:32:04 2015-12-23)[nerv] info: create layer: sigmoidL1 +(16:32:04 2015-12-23)[nerv] info: create layer: combinerL1 +(16:32:04 2015-12-23)[nerv] info: create layer: selectL1 +(16:32:04 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(16:32:04 2015-12-23)[nerv] info: create layer: outputL +(16:32:04 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(16:32:04 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(16:32:04 2015-12-23)[nerv] info: create layer: softmaxL +(16:32:04 2015-12-23)[nerv] info: create layer: recurrentL1 +(16:32:04 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(16:32:04 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER11 LR1]: preparing layers end. [SCHEDULER ITER11 LR1]: Generate and initing TNN ... +(16:32:04 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -2762,253 +3133,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(16:32:04 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:32:04 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:32:04 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:32:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:32:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:32:04 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER11 LR1]: Initing TNN end. ===ITERATION 11 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER11 LR1]: 40092 words processed Wed Nov 18 10:57:27 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.171852. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47361 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76387 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.30974 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58969 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11078 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12087 clock time -[SCHEDULER ITER11 LR1]: 80099 words processed Wed Nov 18 10:57:39 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.172153. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47191 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74498 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24634 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56602 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09802 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11974 clock time -[SCHEDULER ITER11 LR1]: 120004 words processed Wed Nov 18 10:57:51 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.168680. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47058 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74499 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25052 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56585 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09930 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11985 clock time -[SCHEDULER ITER11 LR1]: 160114 words processed Wed Nov 18 10:58:03 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.171976. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47374 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75961 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.30222 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58686 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11161 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12051 clock time -[SCHEDULER ITER11 LR1]: 200066 words processed Wed Nov 18 10:58:15 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.173569. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46822 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76367 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.29753 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58557 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10574 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12124 clock time -[SCHEDULER ITER11 LR1]: 240045 words processed Wed Nov 18 10:58:27 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.170281. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47443 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77754 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.34750 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60093 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11129 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12451 clock time -[SCHEDULER ITER11 LR1]: 280057 words processed Wed Nov 18 10:58:39 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.167366. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48904 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.83714 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.52092 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65348 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13578 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12800 clock time -[SCHEDULER ITER11 LR1]: 320106 words processed Wed Nov 18 10:58:51 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.165503. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47237 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.78810 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.36706 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60854 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11196 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12336 clock time -[SCHEDULER ITER11 LR1]: 360024 words processed Wed Nov 18 10:59:03 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.164444. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47031 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76536 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.30907 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58333 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11299 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12230 clock time -[SCHEDULER ITER11 LR1]: 400089 words processed Wed Nov 18 10:59:15 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.164365. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46607 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75789 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.28348 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58436 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10448 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12046 clock time -[SCHEDULER ITER11 LR1]: 440067 words processed Wed Nov 18 10:59:27 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.165207. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46395 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74732 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24853 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56774 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10194 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12001 clock time -[SCHEDULER ITER11 LR1]: 480051 words processed Wed Nov 18 10:59:39 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.165767. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46867 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.78041 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.33160 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59420 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10951 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12351 clock time -[SCHEDULER ITER11 LR1]: 520140 words processed Wed Nov 18 10:59:51 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.165936. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47031 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.78899 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.36490 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60669 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11743 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12351 clock time -[SCHEDULER ITER11 LR1]: 560132 words processed Wed Nov 18 11:00:03 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.165982. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46470 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75492 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27104 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57423 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10717 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12081 clock time -[SCHEDULER ITER11 LR1]: 600118 words processed Wed Nov 18 11:00:15 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.165231. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46637 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76577 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.30139 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58660 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10742 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12180 clock time -[SCHEDULER ITER11 LR1]: 640090 words processed Wed Nov 18 11:00:27 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.164055. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46516 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76147 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.29472 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10912 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12261 clock time -[SCHEDULER ITER11 LR1]: 680075 words processed Wed Nov 18 11:00:39 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.163994. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46916 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77237 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.32016 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59077 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11044 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12238 clock time -[SCHEDULER ITER11 LR1]: 720043 words processed Wed Nov 18 11:00:51 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.164189. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46639 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77766 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.33385 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59446 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11466 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12254 clock time -[SCHEDULER ITER11 LR1]: 760012 words processed Wed Nov 18 11:01:03 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.163842. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46357 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75535 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26937 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57476 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10461 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12015 clock time -[SCHEDULER ITER11 LR1]: 800113 words processed Wed Nov 18 11:01:15 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.164110. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77157 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.33094 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59946 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11348 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12186 clock time -[SCHEDULER ITER11 LR1]: 840089 words processed Wed Nov 18 11:01:27 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.163404. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47160 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.78736 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.34813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10880 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12376 clock time -[SCHEDULER ITER11 LR1]: 880052 words processed Wed Nov 18 11:01:39 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.163566. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47440 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.80264 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.37384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59985 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11111 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12318 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:32:04 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:32:04 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:32:04 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:32:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:32:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:32:04 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:32:04 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER11 LR1]: 40092 words processed Wed Dec 23 16:32:20 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.367310. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48069 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96432 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75299 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77241 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11451 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14714 clock time +[SCHEDULER ITER11 LR1]: 80099 words processed Wed Dec 23 16:32:36 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.283860. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47988 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93322 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67871 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74900 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10044 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14645 clock time +[SCHEDULER ITER11 LR1]: 120004 words processed Wed Dec 23 16:32:52 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.249194. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47840 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93382 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68619 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75133 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10112 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14687 clock time +[SCHEDULER ITER11 LR1]: 160114 words processed Wed Dec 23 16:33:08 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.236106. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47947 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95668 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73751 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77309 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10818 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14767 clock time +[SCHEDULER ITER11 LR1]: 200066 words processed Wed Dec 23 16:33:24 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.226497. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47411 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93889 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69018 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75655 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10106 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14702 clock time +[SCHEDULER ITER11 LR1]: 240045 words processed Wed Dec 23 16:33:40 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.215354. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47516 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93280 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67717 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75205 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09979 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14653 clock time +[SCHEDULER ITER11 LR1]: 280057 words processed Wed Dec 23 16:33:56 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.206486. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95563 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73347 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77223 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10693 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14726 clock time +[SCHEDULER ITER11 LR1]: 320106 words processed Wed Dec 23 16:34:12 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.200216. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47363 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96036 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73808 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77243 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10800 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14769 clock time +[SCHEDULER ITER11 LR1]: 360024 words processed Wed Dec 23 16:34:28 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.196130. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46950 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93248 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67341 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74983 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10062 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14715 clock time +[SCHEDULER ITER11 LR1]: 400089 words processed Wed Dec 23 16:34:44 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.191623. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47111 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95493 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72399 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76963 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10767 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14742 clock time +[SCHEDULER ITER11 LR1]: 440067 words processed Wed Dec 23 16:35:00 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.189886. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47381 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96987 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74004 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76680 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10472 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14969 clock time +[SCHEDULER ITER11 LR1]: 480051 words processed Wed Dec 23 16:35:16 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.188613. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46973 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94060 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68167 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75270 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09989 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14783 clock time +[SCHEDULER ITER11 LR1]: 520140 words processed Wed Dec 23 16:35:32 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.187514. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47156 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95708 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73358 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77418 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10772 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14829 clock time +[SCHEDULER ITER11 LR1]: 560132 words processed Wed Dec 23 16:35:48 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.185937. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47072 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93481 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67499 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75227 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09932 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14741 clock time +[SCHEDULER ITER11 LR1]: 600118 words processed Wed Dec 23 16:36:04 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.182759. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47002 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93602 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68234 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75634 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10118 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14754 clock time +[SCHEDULER ITER11 LR1]: 640090 words processed Wed Dec 23 16:36:20 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.180637. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47042 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94059 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68509 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75399 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10014 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14789 clock time +[SCHEDULER ITER11 LR1]: 680075 words processed Wed Dec 23 16:36:36 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.179941. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47009 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94393 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69489 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10096 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14773 clock time +[SCHEDULER ITER11 LR1]: 720043 words processed Wed Dec 23 16:36:52 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.179324. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46955 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94175 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69644 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76088 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10176 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14812 clock time +[SCHEDULER ITER11 LR1]: 760012 words processed Wed Dec 23 16:37:08 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.178421. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46982 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94887 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70384 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75987 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10296 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14779 clock time +[SCHEDULER ITER11 LR1]: 800113 words processed Wed Dec 23 16:37:24 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.177858. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47140 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95507 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72776 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77311 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10641 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14775 clock time +[SCHEDULER ITER11 LR1]: 840089 words processed Wed Dec 23 16:37:40 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.176342. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46888 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93251 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66860 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09858 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14684 clock time +[SCHEDULER ITER11 LR1]: 880052 words processed Wed Dec 23 16:37:56 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.175856. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46915 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93449 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67515 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75270 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09948 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14713 clock time [LOG]LMSeqReader: file expires, closing. +(16:37:59 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER11 LR1]: Displaying result: -[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 162.77539106927> <PPL_OOV 145.7641676188> <LOGP -2011305.9620894> -[SCHEDULER ITER11 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 167.45369886632> <PPL_OOV 149.92649649186> <LOGP -2022672.618953> +[SCHEDULER ITER11 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER11 LR1]: shuffling training file ===PEEK ON TEST 11=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER11 LR1]: 40087 words processed Wed Nov 18 11:01:49 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.217961. - [global_conf.timer]: time spent on tnn_beforeprocess:0.52642 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95365 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.73699 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12766 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:38:01 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:38:01 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:38:01 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:38:01 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:38:01 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:38:01 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:38:01 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER11 LR1]: 40087 words processed Wed Dec 23 16:38:09 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.214222. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49422 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12884 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.88738 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14439 clock time [LOG]LMSeqReader: file expires, closing. +(16:38:17 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER11 LR1]: Displaying result: -[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 190.48374176169> <PPL_OOV 165.48555249968> <LOGP -182892.39374501> +[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 186.63895065514> <PPL_OOV 163.48384105363> <LOGP -182456.7304768> [SCHEDULER ITER11 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 11=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER11 LR1]: 40095 words processed Wed Nov 18 11:01:58 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.279941. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46840 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74652 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.43936 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11425 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:38:17 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:38:17 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:38:17 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:38:17 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:38:17 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:38:17 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:38:17 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER11 LR1]: 40095 words processed Wed Dec 23 16:38:25 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.276015. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49291 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17428 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92998 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14268 clock time [LOG]LMSeqReader: file expires, closing. +(16:38:31 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER11 LR1]: Displaying result: -[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 195.54149127002> <PPL_OOV 174.33505652113> <LOGP -165324.53745505> +[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 192.51625883617> <PPL_OOV 172.80450858571> <LOGP -165042.06191345> [SCHEDULER ITER11 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER11 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.11... +[SCHEDULER ITER11 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.11... [SCHEDULER ITER12 LR1]: preparing parameters... -[SCHEDULER ITER12 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.11... +[SCHEDULER ITER12 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.11... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1072320 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3669 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1075977 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120060 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35691290 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1191191 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35807569 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35829535 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70352633 +reading chunk 5 from 70376832 [SCHEDULER ITER12 LR1]: preparing parameters end. [SCHEDULER ITER12 LR1]: preparing layers... -(11:02:07 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:02:07 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:02:07 2015-11-18)[nerv] info: create layer: combinerL1 -(11:02:07 2015-11-18)[nerv] info: create layer: outputL -(11:02:07 2015-11-18)[nerv] info: create layer: softmaxL -(11:02:07 2015-11-18)[nerv] info: create layer: selectL1 +(16:38:37 2015-12-23)[nerv] info: create layer: sigmoidL1 +(16:38:37 2015-12-23)[nerv] info: create layer: combinerL1 +(16:38:37 2015-12-23)[nerv] info: create layer: selectL1 +(16:38:37 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(16:38:37 2015-12-23)[nerv] info: create layer: outputL +(16:38:37 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(16:38:37 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(16:38:37 2015-12-23)[nerv] info: create layer: softmaxL +(16:38:37 2015-12-23)[nerv] info: create layer: recurrentL1 +(16:38:37 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(16:38:37 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER12 LR1]: preparing layers end. [SCHEDULER ITER12 LR1]: Generate and initing TNN ... +(16:38:37 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -3024,253 +3431,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(16:38:37 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:38:37 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:38:37 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:38:37 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:38:37 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:38:37 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER12 LR1]: Initing TNN end. ===ITERATION 12 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER12 LR1]: 40092 words processed Wed Nov 18 11:02:18 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.161409. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46804 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73464 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25376 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58354 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10334 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11784 clock time -[SCHEDULER ITER12 LR1]: 80099 words processed Wed Nov 18 11:02:30 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.160069. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73095 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22902 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57024 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09515 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11827 clock time -[SCHEDULER ITER12 LR1]: 120004 words processed Wed Nov 18 11:02:42 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.156185. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46643 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73770 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22659 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56344 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09182 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11841 clock time -[SCHEDULER ITER12 LR1]: 160114 words processed Wed Nov 18 11:02:54 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.159227. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46840 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75529 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.28682 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59150 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10272 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11974 clock time -[SCHEDULER ITER12 LR1]: 200066 words processed Wed Nov 18 11:03:06 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.160588. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46305 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73458 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.22120 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56632 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09245 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11786 clock time -[SCHEDULER ITER12 LR1]: 240045 words processed Wed Nov 18 11:03:18 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.158727. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46127 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72432 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20139 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56202 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09165 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11760 clock time -[SCHEDULER ITER12 LR1]: 280057 words processed Wed Nov 18 11:03:30 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.156116. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46421 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73845 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24876 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57930 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10000 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11791 clock time -[SCHEDULER ITER12 LR1]: 320106 words processed Wed Nov 18 11:03:42 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.154215. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46172 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74409 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25331 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58024 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09971 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11838 clock time -[SCHEDULER ITER12 LR1]: 360024 words processed Wed Nov 18 11:03:54 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.153248. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46075 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75659 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58098 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10165 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12113 clock time -[SCHEDULER ITER12 LR1]: 400089 words processed Wed Nov 18 11:04:06 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.152203. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47339 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.78152 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.35988 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60790 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11952 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12028 clock time -[SCHEDULER ITER12 LR1]: 440067 words processed Wed Nov 18 11:04:18 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.153060. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46945 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77317 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.32184 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58860 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11549 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12029 clock time -[SCHEDULER ITER12 LR1]: 480051 words processed Wed Nov 18 11:04:30 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.153656. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46704 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77470 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.32119 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11553 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12137 clock time -[SCHEDULER ITER12 LR1]: 520140 words processed Wed Nov 18 11:04:42 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.153706. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.78471 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.35474 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60051 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12341 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12213 clock time -[SCHEDULER ITER12 LR1]: 560132 words processed Wed Nov 18 11:04:54 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.153747. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46748 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77519 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.32001 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58846 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11321 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12274 clock time -[SCHEDULER ITER12 LR1]: 600118 words processed Wed Nov 18 11:05:06 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.152182. +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:38:37 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:38:37 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:38:37 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:38:37 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:38:37 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:38:37 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:38:37 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER12 LR1]: 40092 words processed Wed Dec 23 16:38:53 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.177369. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48530 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99720 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.80172 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77644 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11598 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15003 clock time +[SCHEDULER ITER12 LR1]: 80099 words processed Wed Dec 23 16:39:09 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.170240. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48371 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96620 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72449 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74955 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10342 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14934 clock time +[SCHEDULER ITER12 LR1]: 120004 words processed Wed Dec 23 16:39:25 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.162958. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47891 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93335 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67285 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74001 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09948 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14757 clock time +[SCHEDULER ITER12 LR1]: 160114 words processed Wed Dec 23 16:39:41 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.164384. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47867 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95317 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71693 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75960 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10649 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14789 clock time +[SCHEDULER ITER12 LR1]: 200066 words processed Wed Dec 23 16:39:57 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.164061. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47442 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93882 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66770 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09699 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14765 clock time +[SCHEDULER ITER12 LR1]: 240045 words processed Wed Dec 23 16:40:13 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.159427. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47453 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94249 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74647 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10123 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14816 clock time +[SCHEDULER ITER12 LR1]: 280057 words processed Wed Dec 23 16:40:29 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.155285. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47643 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95340 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71401 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75789 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10451 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14779 clock time +[SCHEDULER ITER12 LR1]: 320106 words processed Wed Dec 23 16:40:45 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.152736. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95381 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71510 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76004 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10601 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14825 clock time +[SCHEDULER ITER12 LR1]: 360024 words processed Wed Dec 23 16:41:01 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.151624. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47023 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93542 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66544 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09919 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14800 clock time +[SCHEDULER ITER12 LR1]: 400089 words processed Wed Dec 23 16:41:17 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.149675. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47148 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95878 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71716 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76002 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10544 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14869 clock time +[SCHEDULER ITER12 LR1]: 440067 words processed Wed Dec 23 16:41:33 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.150093. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46921 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93312 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65385 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73670 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09782 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14727 clock time +[SCHEDULER ITER12 LR1]: 480051 words processed Wed Dec 23 16:41:49 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.150669. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46989 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93279 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65216 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73707 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09757 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14722 clock time +[SCHEDULER ITER12 LR1]: 520140 words processed Wed Dec 23 16:42:05 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.150736. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47194 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95931 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71585 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75842 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10458 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14826 clock time +[SCHEDULER ITER12 LR1]: 560132 words processed Wed Dec 23 16:42:21 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.150550. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46995 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93290 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65263 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73618 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09738 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14760 clock time +[SCHEDULER ITER12 LR1]: 600118 words processed Wed Dec 23 16:42:37 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.149039. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46931 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93249 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65010 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73576 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09644 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14716 clock time +[SCHEDULER ITER12 LR1]: 640090 words processed Wed Dec 23 16:42:53 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.147780. [global_conf.timer]: time spent on tnn_beforeprocess:0.46889 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77258 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.32317 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58892 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11644 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12163 clock time -[SCHEDULER ITER12 LR1]: 640090 words processed Wed Nov 18 11:05:18 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.151157. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46399 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77629 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.32001 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58683 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11419 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12488 clock time -[SCHEDULER ITER12 LR1]: 680075 words processed Wed Nov 18 11:05:30 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.150658. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46552 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.78214 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.33472 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59564 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11356 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12239 clock time -[SCHEDULER ITER12 LR1]: 720043 words processed Wed Nov 18 11:05:42 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.150768. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46935 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77457 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.32346 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59136 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11228 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12066 clock time -[SCHEDULER ITER12 LR1]: 760012 words processed Wed Nov 18 11:05:54 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.150376. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46493 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77122 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.31371 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58771 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11378 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12127 clock time -[SCHEDULER ITER12 LR1]: 800113 words processed Wed Nov 18 11:06:06 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.150726. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47058 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.78021 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.35393 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60839 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11575 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12193 clock time -[SCHEDULER ITER12 LR1]: 840089 words processed Wed Nov 18 11:06:18 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.150092. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45713 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.72990 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.20382 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56355 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09099 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11758 clock time -[SCHEDULER ITER12 LR1]: 880052 words processed Wed Nov 18 11:06:30 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.150286. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46365 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77542 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.32099 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59537 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10777 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12332 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93381 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73609 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09604 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14752 clock time +[SCHEDULER ITER12 LR1]: 680075 words processed Wed Dec 23 16:43:09 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.147213. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46998 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94151 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67319 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74408 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09962 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14821 clock time +[SCHEDULER ITER12 LR1]: 720043 words processed Wed Dec 23 16:43:25 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.147182. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46998 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96297 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70080 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74465 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10176 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14815 clock time +[SCHEDULER ITER12 LR1]: 760012 words processed Wed Dec 23 16:43:41 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.148069. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46961 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93401 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65840 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73980 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09634 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14761 clock time +[SCHEDULER ITER12 LR1]: 800113 words processed Wed Dec 23 16:43:57 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.148704. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47070 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95393 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75908 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10514 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14867 clock time +[SCHEDULER ITER12 LR1]: 840089 words processed Wed Dec 23 16:44:13 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.148060. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46907 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93503 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65676 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73822 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09723 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14756 clock time +[SCHEDULER ITER12 LR1]: 880052 words processed Wed Dec 23 16:44:29 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.148327. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46971 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93295 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65445 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73851 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09495 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14767 clock time [LOG]LMSeqReader: file expires, closing. +(16:44:32 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER12 LR1]: Displaying result: -[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 157.67304410435> <PPL_OOV 141.37325038058> <LOGP -1998957.7352271> -[SCHEDULER ITER12 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 156.91708874811> <PPL_OOV 140.75651809159> <LOGP -1997192.6984916> +[SCHEDULER ITER12 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER12 LR1]: shuffling training file ===PEEK ON TEST 12=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER12 LR1]: 40087 words processed Wed Nov 18 11:06:39 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.213910. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47627 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.79067 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.49743 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11713 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:44:34 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:44:34 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:44:34 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:44:34 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:44:34 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:44:34 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:44:34 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER12 LR1]: 40087 words processed Wed Dec 23 16:44:42 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.208133. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49638 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.14576 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.90732 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14528 clock time [LOG]LMSeqReader: file expires, closing. +(16:44:50 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER12 LR1]: Displaying result: -[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 188.82922250456> <PPL_OOV 163.82151680349> <LOGP -182530.59680635> +[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 183.27056607718> <PPL_OOV 161.23248188013> <LOGP -181960.31280688> [SCHEDULER ITER12 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 12=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER12 LR1]: 40095 words processed Wed Nov 18 11:06:48 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.276348. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47279 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76275 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.46183 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11499 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:44:50 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:44:50 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:44:50 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:44:50 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:44:50 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:44:50 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:44:50 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER12 LR1]: 40095 words processed Wed Dec 23 16:44:58 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.271286. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49182 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17920 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.93392 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14319 clock time [LOG]LMSeqReader: file expires, closing. +(16:45:04 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER12 LR1]: Displaying result: -[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 194.0611897081> <PPL_OOV 172.80269357813> <LOGP -165041.72545543> +[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 189.22591781162> <PPL_OOV 170.45488531847> <LOGP -164603.51324079> [SCHEDULER ITER12 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER12 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.12... +[SCHEDULER ITER12 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.12... [SCHEDULER ITER13 LR1]: preparing parameters... -[SCHEDULER ITER13 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.12... +[SCHEDULER ITER13 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.12... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34545115 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3669 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34548772 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120063 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69171458 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1190556 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69287727 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35835448 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70359323 +reading chunk 5 from 70382428 [SCHEDULER ITER13 LR1]: preparing parameters end. [SCHEDULER ITER13 LR1]: preparing layers... -(11:06:57 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:06:57 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:06:57 2015-11-18)[nerv] info: create layer: combinerL1 -(11:06:57 2015-11-18)[nerv] info: create layer: outputL -(11:06:57 2015-11-18)[nerv] info: create layer: softmaxL -(11:06:57 2015-11-18)[nerv] info: create layer: selectL1 +(16:45:10 2015-12-23)[nerv] info: create layer: sigmoidL1 +(16:45:10 2015-12-23)[nerv] info: create layer: combinerL1 +(16:45:10 2015-12-23)[nerv] info: create layer: selectL1 +(16:45:10 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(16:45:10 2015-12-23)[nerv] info: create layer: outputL +(16:45:10 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(16:45:10 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(16:45:10 2015-12-23)[nerv] info: create layer: softmaxL +(16:45:10 2015-12-23)[nerv] info: create layer: recurrentL1 +(16:45:10 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(16:45:10 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER13 LR1]: preparing layers end. [SCHEDULER ITER13 LR1]: Generate and initing TNN ... +(16:45:10 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -3286,253 +3729,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(16:45:10 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:45:10 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:45:10 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:45:10 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:45:10 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:45:10 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER13 LR1]: Initing TNN end. ===ITERATION 13 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER13 LR1]: 40092 words processed Wed Nov 18 11:07:08 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.148363. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47396 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76596 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.32880 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60443 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11453 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12052 clock time -[SCHEDULER ITER13 LR1]: 80099 words processed Wed Nov 18 11:07:20 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.147670. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47176 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74437 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25697 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57814 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09877 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11998 clock time -[SCHEDULER ITER13 LR1]: 120004 words processed Wed Nov 18 11:07:32 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.144747. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74474 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57923 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09952 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11961 clock time -[SCHEDULER ITER13 LR1]: 160114 words processed Wed Nov 18 11:07:44 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.148082. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47093 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75136 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.28135 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59058 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10353 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11983 clock time -[SCHEDULER ITER13 LR1]: 200066 words processed Wed Nov 18 11:07:56 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.149260. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46573 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73982 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23573 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57354 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09275 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11962 clock time -[SCHEDULER ITER13 LR1]: 240045 words processed Wed Nov 18 11:08:08 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.145787. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46671 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74277 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24641 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57707 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09559 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11984 clock time -[SCHEDULER ITER13 LR1]: 280057 words processed Wed Nov 18 11:08:20 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.142989. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46930 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75368 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.29202 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59411 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10637 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11980 clock time -[SCHEDULER ITER13 LR1]: 320106 words processed Wed Nov 18 11:08:32 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.141522. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46494 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74790 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27671 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58973 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10661 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11994 clock time -[SCHEDULER ITER13 LR1]: 360024 words processed Wed Nov 18 11:08:44 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.140712. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46299 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73963 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23861 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57339 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09706 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11945 clock time -[SCHEDULER ITER13 LR1]: 400089 words processed Wed Nov 18 11:08:56 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.139579. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46577 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75574 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.28922 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59270 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10744 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12032 clock time -[SCHEDULER ITER13 LR1]: 440067 words processed Wed Nov 18 11:09:08 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.140484. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46466 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75020 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26588 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58459 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09975 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12030 clock time -[SCHEDULER ITER13 LR1]: 480051 words processed Wed Nov 18 11:09:20 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.141125. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46269 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73945 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23088 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57175 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09546 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11945 clock time -[SCHEDULER ITER13 LR1]: 520140 words processed Wed Nov 18 11:09:32 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.141141. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46447 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74927 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27264 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10330 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11981 clock time -[SCHEDULER ITER13 LR1]: 560132 words processed Wed Nov 18 11:09:44 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.141279. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46258 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74012 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23609 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57320 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09816 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11973 clock time -[SCHEDULER ITER13 LR1]: 600118 words processed Wed Nov 18 11:09:56 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.139978. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46141 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.73807 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23119 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09558 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11915 clock time -[SCHEDULER ITER13 LR1]: 640090 words processed Wed Nov 18 11:10:08 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.139004. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46270 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74287 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.24469 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57710 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09776 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11997 clock time -[SCHEDULER ITER13 LR1]: 680075 words processed Wed Nov 18 11:10:20 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.138886. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46222 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74030 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.23620 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57425 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09672 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11966 clock time -[SCHEDULER ITER13 LR1]: 720043 words processed Wed Nov 18 11:10:32 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.139161. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46158 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74596 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.25004 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57909 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09737 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12003 clock time -[SCHEDULER ITER13 LR1]: 760012 words processed Wed Nov 18 11:10:44 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.138622. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46481 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75269 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27437 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58719 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10122 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12073 clock time -[SCHEDULER ITER13 LR1]: 800113 words processed Wed Nov 18 11:10:56 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.139039. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75190 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27988 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59340 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10426 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11990 clock time -[SCHEDULER ITER13 LR1]: 840089 words processed Wed Nov 18 11:11:08 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.138421. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46461 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.75640 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.27998 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58710 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10361 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12059 clock time -[SCHEDULER ITER13 LR1]: 880052 words processed Wed Nov 18 11:11:20 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.138626. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46274 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.74625 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.26140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58278 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10143 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12049 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:45:10 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:45:10 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:45:10 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:45:10 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:45:10 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:45:10 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:45:10 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER13 LR1]: 40092 words processed Wed Dec 23 16:45:26 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.151557. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48556 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00768 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.82329 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.79028 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11518 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14941 clock time +[SCHEDULER ITER13 LR1]: 80099 words processed Wed Dec 23 16:45:42 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.148593. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93635 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67630 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74825 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09615 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14733 clock time +[SCHEDULER ITER13 LR1]: 120004 words processed Wed Dec 23 16:45:58 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.142573. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47956 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95238 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71526 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75931 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10160 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14806 clock time +[SCHEDULER ITER13 LR1]: 160114 words processed Wed Dec 23 16:46:14 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.145183. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47945 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96506 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74203 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77247 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10487 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14863 clock time +[SCHEDULER ITER13 LR1]: 200066 words processed Wed Dec 23 16:46:30 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.145219. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47743 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96389 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72047 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75940 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09835 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14901 clock time +[SCHEDULER ITER13 LR1]: 240045 words processed Wed Dec 23 16:46:46 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.140724. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47628 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95250 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70346 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75652 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09844 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14877 clock time +[SCHEDULER ITER13 LR1]: 280057 words processed Wed Dec 23 16:47:02 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.136964. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47763 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98559 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.77186 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77689 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10654 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14899 clock time +[SCHEDULER ITER13 LR1]: 320106 words processed Wed Dec 23 16:47:18 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.134882. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47441 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97003 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74918 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77636 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10517 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14846 clock time +[SCHEDULER ITER13 LR1]: 360024 words processed Wed Dec 23 16:47:34 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.133977. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47081 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95131 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69824 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75539 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09880 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14814 clock time +[SCHEDULER ITER13 LR1]: 400089 words processed Wed Dec 23 16:47:50 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.132044. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47150 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95900 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72446 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77161 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10236 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14840 clock time +[SCHEDULER ITER13 LR1]: 440067 words processed Wed Dec 23 16:48:06 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.132789. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46922 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93905 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66770 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74727 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09603 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14734 clock time +[SCHEDULER ITER13 LR1]: 480051 words processed Wed Dec 23 16:48:22 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.133591. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46966 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93578 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66209 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74835 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09403 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14729 clock time +[SCHEDULER ITER13 LR1]: 520140 words processed Wed Dec 23 16:48:38 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.133658. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47110 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95785 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71911 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76875 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10198 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14799 clock time +[SCHEDULER ITER13 LR1]: 560132 words processed Wed Dec 23 16:48:54 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.133577. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46941 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93675 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66221 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74621 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09509 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14736 clock time +[SCHEDULER ITER13 LR1]: 600118 words processed Wed Dec 23 16:49:10 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.131650. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46970 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94414 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68313 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75492 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09668 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14827 clock time +[SCHEDULER ITER13 LR1]: 640090 words processed Wed Dec 23 16:49:26 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.130534. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47137 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99019 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.77330 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77766 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10755 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15140 clock time +[SCHEDULER ITER13 LR1]: 680075 words processed Wed Dec 23 16:49:42 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.130114. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47348 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97525 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73554 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09953 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14917 clock time +[SCHEDULER ITER13 LR1]: 720043 words processed Wed Dec 23 16:49:58 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.130252. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46922 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93841 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67717 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75454 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09676 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14763 clock time +[SCHEDULER ITER13 LR1]: 760012 words processed Wed Dec 23 16:50:14 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.130228. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46935 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94069 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67825 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75210 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09805 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14741 clock time +[SCHEDULER ITER13 LR1]: 800113 words processed Wed Dec 23 16:50:30 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.130611. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95900 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76811 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10224 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14775 clock time +[SCHEDULER ITER13 LR1]: 840089 words processed Wed Dec 23 16:50:46 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.130105. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46859 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93682 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66576 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74829 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09643 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14719 clock time +[SCHEDULER ITER13 LR1]: 880052 words processed Wed Dec 23 16:51:02 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.130432. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46984 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93882 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67403 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75146 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09659 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14723 clock time [LOG]LMSeqReader: file expires, closing. +(16:51:05 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER13 LR1]: Displaying result: -[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 153.32287582627> <PPL_OOV 137.63110760163> <LOGP -1988127.4300539> -[SCHEDULER ITER13 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 150.31423154359> <PPL_OOV 135.08488872594> <LOGP -1980588.6161387> +[SCHEDULER ITER13 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER13 LR1]: shuffling training file ===PEEK ON TEST 13=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER13 LR1]: 40087 words processed Wed Nov 18 11:11:29 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.212208. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47274 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.77361 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.47349 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11629 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:51:07 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:51:07 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:51:07 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:51:07 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:51:07 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:51:07 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:51:07 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER13 LR1]: 40087 words processed Wed Dec 23 16:51:15 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.202743. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49226 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12743 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.88167 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14372 clock time [LOG]LMSeqReader: file expires, closing. +(16:51:23 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER13 LR1]: Displaying result: -[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 187.68231696506> <PPL_OOV 163.13076093519> <LOGP -182379.33110377> +[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 181.39777842679> <PPL_OOV 159.02870033992> <LOGP -181467.6262641> [SCHEDULER ITER13 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 13=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER13 LR1]: 40095 words processed Wed Nov 18 11:11:38 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.275537. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47033 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.76167 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.45707 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11498 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:51:23 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:51:23 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:51:23 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:51:23 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:51:23 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:51:23 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:51:23 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER13 LR1]: 40095 words processed Wed Dec 23 16:51:31 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.266534. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49197 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17525 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92827 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14256 clock time [LOG]LMSeqReader: file expires, closing. +(16:51:37 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER13 LR1]: Displaying result: -[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 193.13701510763> <PPL_OOV 172.21964454358> <LOGP -164933.45910557> +[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 187.52376756734> <PPL_OOV 168.36386572361> <LOGP -164208.11803951> [SCHEDULER ITER13 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER13 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.13... +[SCHEDULER ITER13 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.13... [SCHEDULER ITER14 LR1]: preparing parameters... -[SCHEDULER ITER14 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.13... +[SCHEDULER ITER14 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.13... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1070792 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3668 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1074449 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120049 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35705236 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1189896 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35821520 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35841734 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70365868 +reading chunk 5 from 70388223 [SCHEDULER ITER14 LR1]: preparing parameters end. [SCHEDULER ITER14 LR1]: preparing layers... -(11:11:47 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:11:47 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:11:47 2015-11-18)[nerv] info: create layer: combinerL1 -(11:11:47 2015-11-18)[nerv] info: create layer: outputL -(11:11:47 2015-11-18)[nerv] info: create layer: softmaxL -(11:11:47 2015-11-18)[nerv] info: create layer: selectL1 +(16:51:43 2015-12-23)[nerv] info: create layer: sigmoidL1 +(16:51:43 2015-12-23)[nerv] info: create layer: combinerL1 +(16:51:43 2015-12-23)[nerv] info: create layer: selectL1 +(16:51:43 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(16:51:43 2015-12-23)[nerv] info: create layer: outputL +(16:51:43 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(16:51:43 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(16:51:43 2015-12-23)[nerv] info: create layer: softmaxL +(16:51:43 2015-12-23)[nerv] info: create layer: recurrentL1 +(16:51:43 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(16:51:43 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER14 LR1]: preparing layers end. [SCHEDULER ITER14 LR1]: Generate and initing TNN ... +(16:51:43 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -3548,253 +4027,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(16:51:43 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:51:43 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:51:43 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:51:43 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:51:43 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:51:43 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER14 LR1]: Initing TNN end. ===ITERATION 14 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER14 LR1]: 40092 words processed Wed Nov 18 11:11:58 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.150251. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47041 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91534 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45503 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59565 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10641 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11921 clock time -[SCHEDULER ITER14 LR1]: 80099 words processed Wed Nov 18 11:12:10 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.145932. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47148 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90982 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41327 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09285 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11949 clock time -[SCHEDULER ITER14 LR1]: 120004 words processed Wed Nov 18 11:12:22 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.139618. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46764 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90625 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41182 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57618 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09351 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11956 clock time -[SCHEDULER ITER14 LR1]: 160114 words processed Wed Nov 18 11:12:34 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.140890. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46819 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92013 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45375 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59395 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10410 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11966 clock time -[SCHEDULER ITER14 LR1]: 200066 words processed Wed Nov 18 11:12:46 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.141728. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46662 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91171 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41959 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57911 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09551 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12001 clock time -[SCHEDULER ITER14 LR1]: 240045 words processed Wed Nov 18 11:12:58 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.138069. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46583 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90781 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40710 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57292 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09530 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11981 clock time -[SCHEDULER ITER14 LR1]: 280057 words processed Wed Nov 18 11:13:10 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.134603. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47093 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94100 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.50456 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60816 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10783 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12105 clock time -[SCHEDULER ITER14 LR1]: 320106 words processed Wed Nov 18 11:13:22 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.132832. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48034 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.97739 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.59913 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.63278 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12446 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12282 clock time -[SCHEDULER ITER14 LR1]: 360024 words processed Wed Nov 18 11:13:34 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.131575. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96039 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.53597 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60708 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11452 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12285 clock time -[SCHEDULER ITER14 LR1]: 400089 words processed Wed Nov 18 11:13:46 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.130441. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96392 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.55461 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62002 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11852 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12308 clock time -[SCHEDULER ITER14 LR1]: 440067 words processed Wed Nov 18 11:13:58 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.131438. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46898 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95113 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51252 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60323 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11246 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12295 clock time -[SCHEDULER ITER14 LR1]: 480051 words processed Wed Nov 18 11:14:10 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.132096. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47060 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95252 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51159 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60575 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10683 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12303 clock time -[SCHEDULER ITER14 LR1]: 520140 words processed Wed Nov 18 11:14:22 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.132082. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47293 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96465 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.56386 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62102 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12395 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12345 clock time -[SCHEDULER ITER14 LR1]: 560132 words processed Wed Nov 18 11:14:34 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.132168. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46982 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95490 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51700 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60588 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11174 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12232 clock time -[SCHEDULER ITER14 LR1]: 600118 words processed Wed Nov 18 11:14:46 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.130284. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46396 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92872 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45166 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58625 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10272 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12092 clock time -[SCHEDULER ITER14 LR1]: 640090 words processed Wed Nov 18 11:14:58 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.128971. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45861 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89984 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38138 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56940 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09154 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11870 clock time -[SCHEDULER ITER14 LR1]: 680075 words processed Wed Nov 18 11:15:10 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.128578. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46160 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91097 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40787 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09244 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11946 clock time -[SCHEDULER ITER14 LR1]: 720043 words processed Wed Nov 18 11:15:22 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.128705. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46362 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91068 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42144 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58050 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09973 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11951 clock time -[SCHEDULER ITER14 LR1]: 760012 words processed Wed Nov 18 11:15:34 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.128296. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46131 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90420 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39806 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57231 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09548 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11873 clock time -[SCHEDULER ITER14 LR1]: 800113 words processed Wed Nov 18 11:15:46 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.128718. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46169 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91339 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43186 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58808 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10250 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11919 clock time -[SCHEDULER ITER14 LR1]: 840089 words processed Wed Nov 18 11:15:58 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.128157. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46160 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90462 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39951 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57234 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09741 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11859 clock time -[SCHEDULER ITER14 LR1]: 880052 words processed Wed Nov 18 11:16:10 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.128351. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46259 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90621 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40749 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57675 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09544 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11975 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:51:43 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:51:43 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:51:43 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:51:43 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:51:43 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:51:43 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:51:43 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER14 LR1]: 40092 words processed Wed Dec 23 16:51:59 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.137442. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48055 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95943 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74173 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77412 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10977 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14786 clock time +[SCHEDULER ITER14 LR1]: 80099 words processed Wed Dec 23 16:52:15 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.134514. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48105 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93544 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68242 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75230 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09835 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14800 clock time +[SCHEDULER ITER14 LR1]: 120004 words processed Wed Dec 23 16:52:31 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.129100. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47874 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93110 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74901 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09568 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14768 clock time +[SCHEDULER ITER14 LR1]: 160114 words processed Wed Dec 23 16:52:47 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.130866. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48065 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96538 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74639 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77399 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10795 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14868 clock time +[SCHEDULER ITER14 LR1]: 200066 words processed Wed Dec 23 16:53:03 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.131334. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47472 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93269 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67118 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75094 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09755 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14731 clock time +[SCHEDULER ITER14 LR1]: 240045 words processed Wed Dec 23 16:53:19 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.128543. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47443 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92839 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66164 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74868 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09616 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time +[SCHEDULER ITER14 LR1]: 280057 words processed Wed Dec 23 16:53:35 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.124540. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47589 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94927 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71840 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76952 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10489 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14775 clock time +[SCHEDULER ITER14 LR1]: 320106 words processed Wed Dec 23 16:53:51 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.122549. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47355 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95060 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71580 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76758 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10586 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14761 clock time +[SCHEDULER ITER14 LR1]: 360024 words processed Wed Dec 23 16:54:07 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.121913. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47042 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93266 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66602 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74796 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09659 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14793 clock time +[SCHEDULER ITER14 LR1]: 400089 words processed Wed Dec 23 16:54:23 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.120106. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47164 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95684 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72912 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77467 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10619 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14942 clock time +[SCHEDULER ITER14 LR1]: 440067 words processed Wed Dec 23 16:54:39 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.120894. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47023 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93495 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67306 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75384 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09799 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14830 clock time +[SCHEDULER ITER14 LR1]: 480051 words processed Wed Dec 23 16:54:55 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.121724. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46965 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92929 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65639 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74795 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09674 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14713 clock time +[SCHEDULER ITER14 LR1]: 520140 words processed Wed Dec 23 16:55:11 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.123036. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47208 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95883 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72471 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77043 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10414 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14830 clock time +[SCHEDULER ITER14 LR1]: 560132 words processed Wed Dec 23 16:55:27 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.123157. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46984 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93034 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65789 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74760 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09618 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14775 clock time +[SCHEDULER ITER14 LR1]: 600118 words processed Wed Dec 23 16:55:43 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.121644. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46999 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92999 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65993 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74919 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09626 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14760 clock time +[SCHEDULER ITER14 LR1]: 640090 words processed Wed Dec 23 16:55:59 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.120502. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46934 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93008 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65684 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74687 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09717 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14721 clock time +[SCHEDULER ITER14 LR1]: 680075 words processed Wed Dec 23 16:56:15 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.119984. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46943 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93256 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66381 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09670 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14755 clock time +[SCHEDULER ITER14 LR1]: 720043 words processed Wed Dec 23 16:56:31 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.120049. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46964 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93727 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67282 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75160 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09789 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14777 clock time +[SCHEDULER ITER14 LR1]: 760012 words processed Wed Dec 23 16:56:47 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.119625. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46887 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93070 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66441 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75155 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09733 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14751 clock time +[SCHEDULER ITER14 LR1]: 800113 words processed Wed Dec 23 16:57:03 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.119967. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47235 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95538 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72220 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77172 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10465 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14786 clock time +[SCHEDULER ITER14 LR1]: 840089 words processed Wed Dec 23 16:57:19 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.119348. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46921 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93053 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65469 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74712 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09403 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14735 clock time +[SCHEDULER ITER14 LR1]: 880052 words processed Wed Dec 23 16:57:35 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.119672. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46886 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93214 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66572 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75014 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09774 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14812 clock time [LOG]LMSeqReader: file expires, closing. +(16:57:38 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER14 LR1]: Displaying result: -[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 149.5870281095> <PPL_OOV 134.4143543947> <LOGP -1978579.6637335> -[SCHEDULER ITER14 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 146.45382776208> <PPL_OOV 131.77770056239> <LOGP -1970581.7353242> +[SCHEDULER ITER14 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER14 LR1]: shuffling training file ===PEEK ON TEST 14=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER14 LR1]: 40087 words processed Wed Nov 18 11:16:19 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.206408. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91040 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.60288 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11501 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:57:40 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:57:40 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:57:40 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:57:40 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:57:40 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:57:40 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:57:40 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER14 LR1]: 40087 words processed Wed Dec 23 16:57:48 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.197733. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49283 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.13392 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.88802 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14379 clock time [LOG]LMSeqReader: file expires, closing. +(16:57:56 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER14 LR1]: Displaying result: -[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 184.60529835793> <PPL_OOV 160.99729765694> <LOGP -181908.05608115> +[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 179.16833296937> <PPL_OOV 157.10841199912> <LOGP -181032.71975146> [SCHEDULER ITER14 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 14=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER14 LR1]: 40095 words processed Wed Nov 18 11:16:28 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.269856. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47128 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92020 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.61603 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11468 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:57:56 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:57:56 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:57:56 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:57:56 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:57:56 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:57:56 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:57:56 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER14 LR1]: 40095 words processed Wed Dec 23 16:58:04 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.262450. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49315 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.18330 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.93852 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14308 clock time [LOG]LMSeqReader: file expires, closing. +(16:58:10 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER14 LR1]: Displaying result: -[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 189.97648963555> <PPL_OOV 169.84953181086> <LOGP -164489.54675678> +[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 185.30574707622> <PPL_OOV 166.38794303038> <LOGP -163829.94786353> [SCHEDULER ITER14 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER14 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.14... +[SCHEDULER ITER14 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.14... [SCHEDULER ITER15 LR1]: preparing parameters... -[SCHEDULER ITER15 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.14... +[SCHEDULER ITER15 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.14... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34543217 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3668 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34546874 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120050 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69185372 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1189289 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69301640 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35849328 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70371575 +reading chunk 5 from 70394874 [SCHEDULER ITER15 LR1]: preparing parameters end. [SCHEDULER ITER15 LR1]: preparing layers... -(11:16:37 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:16:37 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:16:37 2015-11-18)[nerv] info: create layer: combinerL1 -(11:16:37 2015-11-18)[nerv] info: create layer: outputL -(11:16:37 2015-11-18)[nerv] info: create layer: softmaxL -(11:16:37 2015-11-18)[nerv] info: create layer: selectL1 +(16:58:16 2015-12-23)[nerv] info: create layer: sigmoidL1 +(16:58:16 2015-12-23)[nerv] info: create layer: combinerL1 +(16:58:16 2015-12-23)[nerv] info: create layer: selectL1 +(16:58:16 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(16:58:16 2015-12-23)[nerv] info: create layer: outputL +(16:58:16 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(16:58:16 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(16:58:16 2015-12-23)[nerv] info: create layer: softmaxL +(16:58:16 2015-12-23)[nerv] info: create layer: recurrentL1 +(16:58:16 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(16:58:16 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER15 LR1]: preparing layers end. [SCHEDULER ITER15 LR1]: Generate and initing TNN ... +(16:58:16 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -3810,253 +4325,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(16:58:16 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:58:16 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:58:16 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:58:16 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:58:16 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:58:16 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER15 LR1]: Initing TNN end. ===ITERATION 15 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER15 LR1]: 40092 words processed Wed Nov 18 11:16:48 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.126334. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47281 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90872 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45864 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59812 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10804 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11982 clock time -[SCHEDULER ITER15 LR1]: 80099 words processed Wed Nov 18 11:17:00 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.126555. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47283 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89088 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40018 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09452 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11950 clock time -[SCHEDULER ITER15 LR1]: 120004 words processed Wed Nov 18 11:17:12 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.123932. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47132 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89178 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40940 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57686 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09738 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11959 clock time -[SCHEDULER ITER15 LR1]: 160114 words processed Wed Nov 18 11:17:24 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.127397. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90734 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44979 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59540 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10475 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11992 clock time -[SCHEDULER ITER15 LR1]: 200066 words processed Wed Nov 18 11:17:36 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.128886. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46510 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89313 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40416 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57835 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09820 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11952 clock time -[SCHEDULER ITER15 LR1]: 240045 words processed Wed Nov 18 11:17:48 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.125460. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46725 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89147 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39646 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57557 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09409 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11943 clock time -[SCHEDULER ITER15 LR1]: 280057 words processed Wed Nov 18 11:18:00 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.122478. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47008 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90419 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44927 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59518 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10608 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12001 clock time -[SCHEDULER ITER15 LR1]: 320106 words processed Wed Nov 18 11:18:12 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.120817. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46478 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90055 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43575 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59445 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10286 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12026 clock time -[SCHEDULER ITER15 LR1]: 360024 words processed Wed Nov 18 11:18:24 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.120126. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46204 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.88698 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38615 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57250 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09479 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11928 clock time -[SCHEDULER ITER15 LR1]: 400089 words processed Wed Nov 18 11:18:36 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.118832. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46424 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89624 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41956 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58714 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10255 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11961 clock time -[SCHEDULER ITER15 LR1]: 440067 words processed Wed Nov 18 11:18:48 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.119879. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46410 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89594 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57877 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09594 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11985 clock time -[SCHEDULER ITER15 LR1]: 480051 words processed Wed Nov 18 11:19:00 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.120634. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46317 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89060 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39291 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57547 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09686 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11966 clock time -[SCHEDULER ITER15 LR1]: 520140 words processed Wed Nov 18 11:19:12 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.120748. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46423 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90075 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43622 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59509 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10480 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12019 clock time -[SCHEDULER ITER15 LR1]: 560132 words processed Wed Nov 18 11:19:24 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.120808. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46609 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89658 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41628 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58231 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10196 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11999 clock time -[SCHEDULER ITER15 LR1]: 600118 words processed Wed Nov 18 11:19:36 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.118991. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46253 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.88829 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39125 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57646 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09666 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11951 clock time -[SCHEDULER ITER15 LR1]: 640090 words processed Wed Nov 18 11:19:48 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.118087. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46408 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90738 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43047 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58424 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10355 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12001 clock time -[SCHEDULER ITER15 LR1]: 680075 words processed Wed Nov 18 11:20:00 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.117886. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46551 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89564 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41294 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58154 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10046 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11980 clock time -[SCHEDULER ITER15 LR1]: 720043 words processed Wed Nov 18 11:20:12 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.118077. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46369 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89339 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40549 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58043 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09661 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11985 clock time -[SCHEDULER ITER15 LR1]: 760012 words processed Wed Nov 18 11:20:24 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.118207. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46190 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.88916 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39454 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57867 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09549 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11958 clock time -[SCHEDULER ITER15 LR1]: 800113 words processed Wed Nov 18 11:20:36 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.118705. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46559 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90859 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59495 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10567 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12053 clock time -[SCHEDULER ITER15 LR1]: 840089 words processed Wed Nov 18 11:20:48 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.118178. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46272 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89479 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40872 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58213 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09935 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12020 clock time -[SCHEDULER ITER15 LR1]: 880052 words processed Wed Nov 18 11:21:00 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.118361. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46266 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.88857 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39067 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57567 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09408 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11984 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(16:58:16 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(16:58:16 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(16:58:16 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(16:58:16 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(16:58:16 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(16:58:16 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(16:58:16 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER15 LR1]: 40092 words processed Wed Dec 23 16:58:32 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.125554. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48090 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96056 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73242 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76208 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10816 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14769 clock time +[SCHEDULER ITER15 LR1]: 80099 words processed Wed Dec 23 16:58:48 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.122392. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48070 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93835 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66851 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73632 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09667 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14715 clock time +[SCHEDULER ITER15 LR1]: 120004 words processed Wed Dec 23 16:59:04 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.117068. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47885 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93643 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67559 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74116 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09816 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14720 clock time +[SCHEDULER ITER15 LR1]: 160114 words processed Wed Dec 23 16:59:20 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.119384. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47889 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95799 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71745 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75727 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10467 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time +[SCHEDULER ITER15 LR1]: 200066 words processed Wed Dec 23 16:59:36 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.119709. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47416 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93499 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66031 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73646 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09733 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14681 clock time +[SCHEDULER ITER15 LR1]: 240045 words processed Wed Dec 23 16:59:52 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.116333. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47391 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93729 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65949 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73543 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09635 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14639 clock time +[SCHEDULER ITER15 LR1]: 280057 words processed Wed Dec 23 17:00:08 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.112568. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47884 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98009 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76921 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77518 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10904 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14842 clock time +[SCHEDULER ITER15 LR1]: 320106 words processed Wed Dec 23 17:00:24 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.110616. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47459 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96569 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73031 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76139 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10609 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14732 clock time +[SCHEDULER ITER15 LR1]: 360024 words processed Wed Dec 23 17:00:40 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.109947. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46993 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93685 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66050 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73649 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09726 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14708 clock time +[SCHEDULER ITER15 LR1]: 400089 words processed Wed Dec 23 17:00:56 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.108064. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47146 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95614 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70749 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75694 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10317 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14739 clock time +[SCHEDULER ITER15 LR1]: 440067 words processed Wed Dec 23 17:01:12 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.108972. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47048 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94322 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67133 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74295 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09683 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14693 clock time +[SCHEDULER ITER15 LR1]: 480051 words processed Wed Dec 23 17:01:28 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.109838. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46983 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93670 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65572 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73771 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09611 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14685 clock time +[SCHEDULER ITER15 LR1]: 520140 words processed Wed Dec 23 17:01:44 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.110561. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47255 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96567 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72923 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76370 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10468 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14833 clock time +[SCHEDULER ITER15 LR1]: 560132 words processed Wed Dec 23 17:02:00 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.110649. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47070 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93756 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65599 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73698 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09469 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14691 clock time +[SCHEDULER ITER15 LR1]: 600118 words processed Wed Dec 23 17:02:16 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.109082. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47082 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94277 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66955 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74387 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09486 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14711 clock time +[SCHEDULER ITER15 LR1]: 640090 words processed Wed Dec 23 17:02:32 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.107947. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46971 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94040 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66111 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73868 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09587 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14704 clock time +[SCHEDULER ITER15 LR1]: 680075 words processed Wed Dec 23 17:02:48 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.107553. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46855 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93567 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65324 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73748 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09556 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14700 clock time +[SCHEDULER ITER15 LR1]: 720043 words processed Wed Dec 23 17:03:04 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.107682. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46940 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94063 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66843 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74313 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09608 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14717 clock time +[SCHEDULER ITER15 LR1]: 760012 words processed Wed Dec 23 17:03:20 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.107385. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46892 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94188 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67671 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74843 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09752 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14760 clock time +[SCHEDULER ITER15 LR1]: 800113 words processed Wed Dec 23 17:03:36 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.107753. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47365 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98016 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75680 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76989 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10874 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14877 clock time +[SCHEDULER ITER15 LR1]: 840089 words processed Wed Dec 23 17:03:52 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.107496. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46851 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93710 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65458 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73661 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09560 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14729 clock time +[SCHEDULER ITER15 LR1]: 880052 words processed Wed Dec 23 17:04:08 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.107886. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93528 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65989 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73883 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09724 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14692 clock time [LOG]LMSeqReader: file expires, closing. +(17:04:11 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER15 LR1]: Displaying result: -[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 146.03546688066> <PPL_OOV 131.3598251194> <LOGP -1969299.4941708> -[SCHEDULER ITER15 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 142.37076880222> <PPL_OOV 128.26094012827> <LOGP -1959661.3827604> +[SCHEDULER ITER15 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER15 LR1]: shuffling training file ===PEEK ON TEST 15=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER15 LR1]: 40087 words processed Wed Nov 18 11:21:09 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.204185. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46840 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90269 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.59680 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11471 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:04:13 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:04:13 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:04:13 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:04:13 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:04:13 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:04:13 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:04:13 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER15 LR1]: 40087 words processed Wed Dec 23 17:04:21 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.194015. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49176 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.10633 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.85994 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14349 clock time [LOG]LMSeqReader: file expires, closing. +(17:04:29 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER15 LR1]: Displaying result: -[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 183.37849337646> <PPL_OOV 160.0011301428> <LOGP -181685.86303108> +[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 177.78465238165> <PPL_OOV 155.69309248801> <LOGP -180708.76210807> [SCHEDULER ITER15 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 15=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER15 LR1]: 40095 words processed Wed Nov 18 11:21:18 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.267028. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46884 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90372 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.59827 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11460 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:04:29 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:04:29 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:04:29 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:04:29 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:04:29 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:04:29 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:04:29 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER15 LR1]: 40095 words processed Wed Dec 23 17:04:37 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.258946. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49105 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17606 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92807 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14270 clock time [LOG]LMSeqReader: file expires, closing. +(17:04:43 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER15 LR1]: Displaying result: -[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 188.55855926685> <PPL_OOV 168.66052507921> <LOGP -164264.51192834> +[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 184.0434088091> <PPL_OOV 165.03066655> <LOGP -163567.56886171> [SCHEDULER ITER15 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER15 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.15... +[SCHEDULER ITER15 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.15... [SCHEDULER ITER16 LR1]: preparing parameters... -[SCHEDULER ITER16 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.15... +[SCHEDULER ITER16 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.15... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1069088 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3668 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1072745 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120055 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35720170 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1188696 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35836453 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35857823 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70378787 +reading chunk 5 from 70402154 [SCHEDULER ITER16 LR1]: preparing parameters end. [SCHEDULER ITER16 LR1]: preparing layers... -(11:21:27 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:21:27 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:21:27 2015-11-18)[nerv] info: create layer: combinerL1 -(11:21:27 2015-11-18)[nerv] info: create layer: outputL -(11:21:27 2015-11-18)[nerv] info: create layer: softmaxL -(11:21:27 2015-11-18)[nerv] info: create layer: selectL1 +(17:04:49 2015-12-23)[nerv] info: create layer: sigmoidL1 +(17:04:49 2015-12-23)[nerv] info: create layer: combinerL1 +(17:04:49 2015-12-23)[nerv] info: create layer: selectL1 +(17:04:49 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(17:04:49 2015-12-23)[nerv] info: create layer: outputL +(17:04:49 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(17:04:49 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(17:04:49 2015-12-23)[nerv] info: create layer: softmaxL +(17:04:49 2015-12-23)[nerv] info: create layer: recurrentL1 +(17:04:49 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(17:04:49 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER16 LR1]: preparing layers end. [SCHEDULER ITER16 LR1]: Generate and initing TNN ... +(17:04:49 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -4072,253 +4623,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(17:04:49 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:04:49 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:04:49 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:04:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:04:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:04:49 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER16 LR1]: Initing TNN end. ===ITERATION 16 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER16 LR1]: 40092 words processed Wed Nov 18 11:21:38 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.115694. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47210 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92260 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47435 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60078 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11007 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11977 clock time -[SCHEDULER ITER16 LR1]: 80099 words processed Wed Nov 18 11:21:50 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.116926. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47316 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91554 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10042 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12021 clock time -[SCHEDULER ITER16 LR1]: 120004 words processed Wed Nov 18 11:22:02 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.113799. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46880 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91035 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42212 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57713 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09679 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11886 clock time -[SCHEDULER ITER16 LR1]: 160114 words processed Wed Nov 18 11:22:14 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.116729. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46849 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91387 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44287 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59014 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10396 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11939 clock time -[SCHEDULER ITER16 LR1]: 200066 words processed Wed Nov 18 11:22:26 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.117457. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46369 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90630 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41076 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57584 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09792 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11954 clock time -[SCHEDULER ITER16 LR1]: 240045 words processed Wed Nov 18 11:22:38 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.114227. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46767 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91646 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43849 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10055 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12030 clock time -[SCHEDULER ITER16 LR1]: 280057 words processed Wed Nov 18 11:22:50 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.111300. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46808 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92176 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46852 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59954 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10617 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12008 clock time -[SCHEDULER ITER16 LR1]: 320106 words processed Wed Nov 18 11:23:02 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.109921. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46519 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92552 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46821 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59674 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10863 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11996 clock time -[SCHEDULER ITER16 LR1]: 360024 words processed Wed Nov 18 11:23:14 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.109011. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46117 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90110 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39845 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57121 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09710 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11923 clock time -[SCHEDULER ITER16 LR1]: 400089 words processed Wed Nov 18 11:23:26 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.107795. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46326 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91323 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43619 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58753 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10437 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11958 clock time -[SCHEDULER ITER16 LR1]: 440067 words processed Wed Nov 18 11:23:38 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.108945. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46048 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90228 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39038 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09327 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11885 clock time -[SCHEDULER ITER16 LR1]: 480051 words processed Wed Nov 18 11:23:50 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.109836. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46156 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90198 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39014 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57049 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09295 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11904 clock time -[SCHEDULER ITER16 LR1]: 520140 words processed Wed Nov 18 11:24:02 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.110070. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46481 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92108 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45601 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59539 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10468 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12008 clock time -[SCHEDULER ITER16 LR1]: 560132 words processed Wed Nov 18 11:24:14 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.110257. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46024 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90350 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39936 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57477 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09656 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11933 clock time -[SCHEDULER ITER16 LR1]: 600118 words processed Wed Nov 18 11:24:26 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.108735. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46120 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90696 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41226 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57848 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09983 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11914 clock time -[SCHEDULER ITER16 LR1]: 640090 words processed Wed Nov 18 11:24:38 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.107910. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46181 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90976 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41403 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57939 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09664 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11954 clock time -[SCHEDULER ITER16 LR1]: 680075 words processed Wed Nov 18 11:24:50 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.108014. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46037 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90094 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39001 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.56999 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09636 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11860 clock time -[SCHEDULER ITER16 LR1]: 720043 words processed Wed Nov 18 11:25:02 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.108294. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46118 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90305 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39579 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57269 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09312 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11935 clock time -[SCHEDULER ITER16 LR1]: 760012 words processed Wed Nov 18 11:25:14 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.107997. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91584 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42419 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09765 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11928 clock time -[SCHEDULER ITER16 LR1]: 800113 words processed Wed Nov 18 11:25:26 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.108498. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46431 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92341 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45787 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59727 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10339 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11947 clock time -[SCHEDULER ITER16 LR1]: 840089 words processed Wed Nov 18 11:25:38 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.107983. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46229 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91069 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42426 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58161 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10309 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11938 clock time -[SCHEDULER ITER16 LR1]: 880052 words processed Wed Nov 18 11:25:50 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.108250. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46124 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91089 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41559 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58013 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:04:49 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:04:49 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:04:49 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:04:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:04:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:04:49 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:04:49 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER16 LR1]: 40092 words processed Wed Dec 23 17:05:05 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.120945. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48418 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97993 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.78231 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78251 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11389 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14877 clock time +[SCHEDULER ITER16 LR1]: 80099 words processed Wed Dec 23 17:05:21 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.114851. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48075 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93018 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66314 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74316 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09555 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14694 clock time +[SCHEDULER ITER16 LR1]: 120004 words processed Wed Dec 23 17:05:37 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.109035. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47893 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92914 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66710 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74519 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09523 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14725 clock time +[SCHEDULER ITER16 LR1]: 160114 words processed Wed Dec 23 17:05:53 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.111924. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48119 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97049 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74772 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77017 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10665 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14825 clock time +[SCHEDULER ITER16 LR1]: 200066 words processed Wed Dec 23 17:06:09 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.111775. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93521 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75162 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09573 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14744 clock time +[SCHEDULER ITER16 LR1]: 240045 words processed Wed Dec 23 17:06:25 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.107381. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47740 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94515 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69092 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75117 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09994 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14780 clock time +[SCHEDULER ITER16 LR1]: 280057 words processed Wed Dec 23 17:06:41 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.103419. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48086 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98961 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.79009 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78423 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10924 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14946 clock time +[SCHEDULER ITER16 LR1]: 320106 words processed Wed Dec 23 17:06:57 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.101482. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95875 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73148 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77097 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10611 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14813 clock time +[SCHEDULER ITER16 LR1]: 360024 words processed Wed Dec 23 17:07:13 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.100691. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47039 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93339 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66808 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74914 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09637 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14726 clock time +[SCHEDULER ITER16 LR1]: 400089 words processed Wed Dec 23 17:07:29 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.098638. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47179 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95217 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70823 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76450 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10163 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14785 clock time +[SCHEDULER ITER16 LR1]: 440067 words processed Wed Dec 23 17:07:45 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.099628. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47024 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93037 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74444 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09490 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14706 clock time +[SCHEDULER ITER16 LR1]: 480051 words processed Wed Dec 23 17:08:01 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.100484. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47046 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93061 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65873 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74790 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09512 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14735 clock time +[SCHEDULER ITER16 LR1]: 520140 words processed Wed Dec 23 17:08:17 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.101553. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47766 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00799 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.81118 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78823 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11117 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15021 clock time +[SCHEDULER ITER16 LR1]: 560132 words processed Wed Dec 23 17:08:33 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.101649. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47601 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97971 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75154 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76920 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10459 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15018 clock time +[SCHEDULER ITER16 LR1]: 600118 words processed Wed Dec 23 17:08:49 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.099943. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47931 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00540 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.80179 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78083 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10903 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15160 clock time +[SCHEDULER ITER16 LR1]: 640090 words processed Wed Dec 23 17:09:05 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.098781. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47286 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95909 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70180 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75262 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09859 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14842 clock time +[SCHEDULER ITER16 LR1]: 680075 words processed Wed Dec 23 17:09:21 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.098352. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93048 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65198 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74287 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09515 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14714 clock time +[SCHEDULER ITER16 LR1]: 720043 words processed Wed Dec 23 17:09:37 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.098346. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48047 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.01923 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.82212 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78345 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10905 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15170 clock time +[SCHEDULER ITER16 LR1]: 760012 words processed Wed Dec 23 17:09:53 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.097682. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47012 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93163 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66991 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75463 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09630 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14761 clock time +[SCHEDULER ITER16 LR1]: 800113 words processed Wed Dec 23 17:10:09 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.098036. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47183 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96732 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73889 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77200 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10638 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14853 clock time +[SCHEDULER ITER16 LR1]: 840089 words processed Wed Dec 23 17:10:25 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.097588. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46907 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93205 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65702 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74471 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09663 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14720 clock time +[SCHEDULER ITER16 LR1]: 880052 words processed Wed Dec 23 17:10:41 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.097978. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93079 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65799 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74563 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09542 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11947 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time [LOG]LMSeqReader: file expires, closing. +(17:10:44 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER16 LR1]: Displaying result: -[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 142.52993551102> <PPL_OOV 128.34432819736> <LOGP -1959923.7705658> -[SCHEDULER ITER16 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 139.01405094196> <PPL_OOV 125.36650711722> <LOGP -1950446.4665655> +[SCHEDULER ITER16 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER16 LR1]: shuffling training file ===PEEK ON TEST 16=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER16 LR1]: 40087 words processed Wed Nov 18 11:25:59 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.199247. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47918 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95565 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.66569 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11720 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:10:46 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:10:46 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:10:46 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:10:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:10:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:10:46 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:10:46 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER16 LR1]: 40087 words processed Wed Dec 23 17:10:54 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.190424. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.14034 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.89801 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14450 clock time [LOG]LMSeqReader: file expires, closing. +(17:11:02 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER16 LR1]: Displaying result: -[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 181.85232657852> <PPL_OOV 158.47519639689> <LOGP -181342.80984868> +[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 176.93478295437> <PPL_OOV 154.62506489551> <LOGP -180462.34173733> [SCHEDULER ITER16 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 16=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER16 LR1]: 40095 words processed Wed Nov 18 11:26:08 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.262718. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47243 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93331 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.63182 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11466 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:11:02 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:11:02 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:11:02 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:11:02 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:11:02 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:11:02 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:11:02 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER16 LR1]: 40095 words processed Wed Dec 23 17:11:10 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.256255. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49350 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17839 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.93185 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14211 clock time [LOG]LMSeqReader: file expires, closing. +(17:11:16 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER16 LR1]: Displaying result: -[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 186.63524327582> <PPL_OOV 166.8116302904> <LOGP -163911.41386819> +[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 182.95953635207> <PPL_OOV 163.73680343083> <LOGP -163315.43164681> [SCHEDULER ITER16 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER16 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.16... +[SCHEDULER ITER16 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.16... [SCHEDULER ITER17 LR1]: preparing parameters... -[SCHEDULER ITER17 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.16... +[SCHEDULER ITER17 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.16... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34541020 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3668 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34544677 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120034 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69201806 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1188035 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69318078 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35868348 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70386500 +reading chunk 5 from 70411797 [SCHEDULER ITER17 LR1]: preparing parameters end. [SCHEDULER ITER17 LR1]: preparing layers... -(11:26:17 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:26:17 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:26:17 2015-11-18)[nerv] info: create layer: combinerL1 -(11:26:17 2015-11-18)[nerv] info: create layer: outputL -(11:26:17 2015-11-18)[nerv] info: create layer: softmaxL -(11:26:17 2015-11-18)[nerv] info: create layer: selectL1 +(17:11:22 2015-12-23)[nerv] info: create layer: sigmoidL1 +(17:11:22 2015-12-23)[nerv] info: create layer: combinerL1 +(17:11:22 2015-12-23)[nerv] info: create layer: selectL1 +(17:11:22 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(17:11:22 2015-12-23)[nerv] info: create layer: outputL +(17:11:22 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(17:11:22 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(17:11:22 2015-12-23)[nerv] info: create layer: softmaxL +(17:11:22 2015-12-23)[nerv] info: create layer: recurrentL1 +(17:11:22 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(17:11:22 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER17 LR1]: preparing layers end. [SCHEDULER ITER17 LR1]: Generate and initing TNN ... +(17:11:22 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -4334,253 +4921,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(17:11:22 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:11:22 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:11:22 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:11:22 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:11:22 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:11:22 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER17 LR1]: Initing TNN end. ===ITERATION 17 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER17 LR1]: 40092 words processed Wed Nov 18 11:26:28 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.108420. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47070 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91633 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46963 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60356 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10864 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11983 clock time -[SCHEDULER ITER17 LR1]: 80099 words processed Wed Nov 18 11:26:40 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.108591. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47065 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89624 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40623 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57839 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09639 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11870 clock time -[SCHEDULER ITER17 LR1]: 120004 words processed Wed Nov 18 11:26:52 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.105430. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90293 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42061 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58116 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09776 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11877 clock time -[SCHEDULER ITER17 LR1]: 160114 words processed Wed Nov 18 11:27:04 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.107876. +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:11:22 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:11:22 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:11:22 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:11:22 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:11:22 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:11:22 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:11:22 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER17 LR1]: 40092 words processed Wed Dec 23 17:11:38 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.104154. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48471 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99230 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.79491 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77849 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11412 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14926 clock time +[SCHEDULER ITER17 LR1]: 80099 words processed Wed Dec 23 17:11:54 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.101718. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48006 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92807 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65623 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73647 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09618 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14707 clock time +[SCHEDULER ITER17 LR1]: 120004 words processed Wed Dec 23 17:12:10 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.096714. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47815 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92844 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66791 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74207 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09829 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14769 clock time +[SCHEDULER ITER17 LR1]: 160114 words processed Wed Dec 23 17:12:26 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.098489. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47883 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94656 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70483 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75746 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10401 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14754 clock time +[SCHEDULER ITER17 LR1]: 200066 words processed Wed Dec 23 17:12:42 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.098656. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47403 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92915 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65438 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73695 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09668 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14727 clock time +[SCHEDULER ITER17 LR1]: 240045 words processed Wed Dec 23 17:12:58 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.094817. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47394 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92825 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65215 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73708 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09658 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14681 clock time +[SCHEDULER ITER17 LR1]: 280057 words processed Wed Dec 23 17:13:14 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.090879. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47881 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97393 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75851 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77288 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10815 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14816 clock time +[SCHEDULER ITER17 LR1]: 320106 words processed Wed Dec 23 17:13:30 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.089281. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47318 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94699 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70277 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75894 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10306 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14735 clock time +[SCHEDULER ITER17 LR1]: 360024 words processed Wed Dec 23 17:13:46 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.088862. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46974 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92676 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64849 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73605 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09652 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14729 clock time +[SCHEDULER ITER17 LR1]: 400089 words processed Wed Dec 23 17:14:02 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.086965. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95067 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70394 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75782 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10425 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14751 clock time +[SCHEDULER ITER17 LR1]: 440067 words processed Wed Dec 23 17:14:18 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.088081. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46941 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92806 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65037 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73955 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09732 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14705 clock time +[SCHEDULER ITER17 LR1]: 480051 words processed Wed Dec 23 17:14:34 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.089046. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46932 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92608 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64298 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73663 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09626 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14698 clock time +[SCHEDULER ITER17 LR1]: 520140 words processed Wed Dec 23 17:14:50 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.089481. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47205 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95116 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70829 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76044 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10374 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14836 clock time +[SCHEDULER ITER17 LR1]: 560132 words processed Wed Dec 23 17:15:06 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.089732. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47069 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92885 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65219 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73983 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09636 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14780 clock time +[SCHEDULER ITER17 LR1]: 600118 words processed Wed Dec 23 17:15:22 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.088077. [global_conf.timer]: time spent on tnn_beforeprocess:0.46971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90966 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44840 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10197 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11979 clock time -[SCHEDULER ITER17 LR1]: 200066 words processed Wed Nov 18 11:27:16 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.109757. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46301 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89580 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40292 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09623 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11921 clock time -[SCHEDULER ITER17 LR1]: 240045 words processed Wed Nov 18 11:27:28 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.106544. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89450 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39429 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57507 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09486 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11897 clock time -[SCHEDULER ITER17 LR1]: 280057 words processed Wed Nov 18 11:27:40 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.103579. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46587 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90502 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43303 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59128 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10095 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11927 clock time -[SCHEDULER ITER17 LR1]: 320106 words processed Wed Nov 18 11:27:52 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.102231. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46324 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90426 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43112 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59322 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10025 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11997 clock time -[SCHEDULER ITER17 LR1]: 360024 words processed Wed Nov 18 11:28:04 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.103825. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46076 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89850 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40003 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57779 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09406 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11884 clock time -[SCHEDULER ITER17 LR1]: 400089 words processed Wed Nov 18 11:28:16 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.102596. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46155 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90705 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43219 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59386 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10104 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11954 clock time -[SCHEDULER ITER17 LR1]: 440067 words processed Wed Nov 18 11:28:28 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.103633. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46145 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89711 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57735 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09579 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11901 clock time -[SCHEDULER ITER17 LR1]: 480051 words processed Wed Nov 18 11:28:40 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.104338. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45965 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89582 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38703 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57387 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09366 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11897 clock time -[SCHEDULER ITER17 LR1]: 520140 words processed Wed Nov 18 11:28:52 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.104375. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46122 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90595 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43166 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59305 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10238 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11945 clock time -[SCHEDULER ITER17 LR1]: 560132 words processed Wed Nov 18 11:29:04 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.104388. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46012 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89565 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39441 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57740 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09621 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11936 clock time -[SCHEDULER ITER17 LR1]: 600118 words processed Wed Nov 18 11:29:16 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.103305. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45987 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89412 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38754 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57428 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09481 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11887 clock time -[SCHEDULER ITER17 LR1]: 640090 words processed Wed Nov 18 11:29:28 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.104230. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45953 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89248 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09555 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11921 clock time -[SCHEDULER ITER17 LR1]: 680075 words processed Wed Nov 18 11:29:40 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.103934. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46068 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89485 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39272 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57792 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09378 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11946 clock time -[SCHEDULER ITER17 LR1]: 720043 words processed Wed Nov 18 11:29:52 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.104033. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89192 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38263 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57281 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09239 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11883 clock time -[SCHEDULER ITER17 LR1]: 760012 words processed Wed Nov 18 11:30:04 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.104624. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46126 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89837 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39959 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57811 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09458 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11908 clock time -[SCHEDULER ITER17 LR1]: 800113 words processed Wed Nov 18 11:30:16 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.105026. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46365 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91993 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46566 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60169 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10782 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11994 clock time -[SCHEDULER ITER17 LR1]: 840089 words processed Wed Nov 18 11:30:28 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.104416. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46128 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90964 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58640 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09750 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12016 clock time -[SCHEDULER ITER17 LR1]: 880052 words processed Wed Nov 18 11:30:40 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.104466. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46401 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91300 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43933 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58860 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10184 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11974 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92689 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64669 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73680 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09747 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14696 clock time +[SCHEDULER ITER17 LR1]: 640090 words processed Wed Dec 23 17:15:38 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.087756. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46920 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92640 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73653 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09600 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14691 clock time +[SCHEDULER ITER17 LR1]: 680075 words processed Wed Dec 23 17:15:54 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.087440. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46858 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92556 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64289 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73685 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09663 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14698 clock time +[SCHEDULER ITER17 LR1]: 720043 words processed Wed Dec 23 17:16:10 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.087562. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46929 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92800 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64914 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73843 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09559 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14734 clock time +[SCHEDULER ITER17 LR1]: 760012 words processed Wed Dec 23 17:16:26 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.086901. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46960 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92899 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65268 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74083 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09528 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14733 clock time +[SCHEDULER ITER17 LR1]: 800113 words processed Wed Dec 23 17:16:42 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.087311. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47095 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94805 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70050 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75774 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10410 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14761 clock time +[SCHEDULER ITER17 LR1]: 840089 words processed Wed Dec 23 17:16:58 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.086705. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46878 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92672 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64519 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73779 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09632 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14710 clock time +[SCHEDULER ITER17 LR1]: 880052 words processed Wed Dec 23 17:17:14 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.087081. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46912 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93033 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65308 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09634 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14737 clock time [LOG]LMSeqReader: file expires, closing. +(17:17:17 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER17 LR1]: Displaying result: -[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 141.19954804009> <PPL_OOV 127.21379650058> <LOGP -1956351.8551459> -[SCHEDULER ITER17 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 135.42659816673> <PPL_OOV 122.25864603049> <LOGP -1940312.1391927> +[SCHEDULER ITER17 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER17 LR1]: shuffling training file ===PEEK ON TEST 17=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER17 LR1]: 40087 words processed Wed Nov 18 11:30:49 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.198889. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91153 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.60373 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11422 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:17:19 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:17:19 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:17:19 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:17:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:17:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:17:19 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:17:19 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER17 LR1]: 40087 words processed Wed Dec 23 17:17:27 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.188704. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49518 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.13380 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.89202 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14354 clock time [LOG]LMSeqReader: file expires, closing. +(17:17:35 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER17 LR1]: Displaying result: -[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 180.70520360784> <PPL_OOV 158.41280661091> <LOGP -181328.71347802> +[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 175.55817495508> <PPL_OOV 153.83315765421> <LOGP -180278.52776374> [SCHEDULER ITER17 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 17=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER17 LR1]: 40095 words processed Wed Nov 18 11:30:58 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.263003. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46879 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90774 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.60026 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11386 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:17:35 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:17:35 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:17:35 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:17:35 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:17:35 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:17:35 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:17:35 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER17 LR1]: 40095 words processed Wed Dec 23 17:17:43 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.254689. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49146 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.16594 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.91864 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14267 clock time [LOG]LMSeqReader: file expires, closing. +(17:17:49 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER17 LR1]: Displaying result: -[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 185.44844722879> <PPL_OOV 166.57217575974> <LOGP -163865.39734015> +[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 181.97442495209> <PPL_OOV 163.1652463831> <LOGP -163203.41628189> [SCHEDULER ITER17 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER17 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.17... +[SCHEDULER ITER17 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.17... [SCHEDULER ITER18 LR1]: preparing parameters... -[SCHEDULER ITER18 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.17... +[SCHEDULER ITER18 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.17... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1067636 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3667 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1071293 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120029 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35738936 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1187413 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35855199 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35879541 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70394860 +reading chunk 5 from 70421857 [SCHEDULER ITER18 LR1]: preparing parameters end. [SCHEDULER ITER18 LR1]: preparing layers... -(11:31:07 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:31:07 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:31:07 2015-11-18)[nerv] info: create layer: combinerL1 -(11:31:07 2015-11-18)[nerv] info: create layer: outputL -(11:31:07 2015-11-18)[nerv] info: create layer: softmaxL -(11:31:07 2015-11-18)[nerv] info: create layer: selectL1 +(17:17:55 2015-12-23)[nerv] info: create layer: sigmoidL1 +(17:17:55 2015-12-23)[nerv] info: create layer: combinerL1 +(17:17:55 2015-12-23)[nerv] info: create layer: selectL1 +(17:17:55 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(17:17:55 2015-12-23)[nerv] info: create layer: outputL +(17:17:55 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(17:17:55 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(17:17:55 2015-12-23)[nerv] info: create layer: softmaxL +(17:17:55 2015-12-23)[nerv] info: create layer: recurrentL1 +(17:17:55 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(17:17:55 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER18 LR1]: preparing layers end. [SCHEDULER ITER18 LR1]: Generate and initing TNN ... +(17:17:55 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -4596,253 +5219,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(17:17:55 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:17:55 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:17:55 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:17:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:17:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:17:55 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER18 LR1]: Initing TNN end. ===ITERATION 18 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER18 LR1]: 40092 words processed Wed Nov 18 11:31:18 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.100858. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47344 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92509 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48658 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60662 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10998 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11994 clock time -[SCHEDULER ITER18 LR1]: 80099 words processed Wed Nov 18 11:31:30 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.100635. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47126 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90330 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41027 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57709 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09284 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11920 clock time -[SCHEDULER ITER18 LR1]: 120004 words processed Wed Nov 18 11:31:42 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.097217. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46894 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90217 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40293 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57342 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09006 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11874 clock time -[SCHEDULER ITER18 LR1]: 160114 words processed Wed Nov 18 11:31:54 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.100555. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46938 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91499 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43863 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58903 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09872 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11905 clock time -[SCHEDULER ITER18 LR1]: 200066 words processed Wed Nov 18 11:32:06 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.101992. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46602 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90144 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39875 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57474 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09039 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11899 clock time -[SCHEDULER ITER18 LR1]: 240045 words processed Wed Nov 18 11:32:18 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.098634. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46616 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90745 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41055 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09234 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11910 clock time -[SCHEDULER ITER18 LR1]: 280057 words processed Wed Nov 18 11:32:30 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.095480. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47852 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96904 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.57313 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62612 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11583 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12273 clock time -[SCHEDULER ITER18 LR1]: 320106 words processed Wed Nov 18 11:32:42 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.093899. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47355 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96676 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.56591 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62358 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11704 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12412 clock time -[SCHEDULER ITER18 LR1]: 360024 words processed Wed Nov 18 11:32:54 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.093222. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47020 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94135 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.49107 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59811 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10305 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12157 clock time -[SCHEDULER ITER18 LR1]: 400089 words processed Wed Nov 18 11:33:06 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.092038. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46395 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91898 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45023 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59541 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10204 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11955 clock time -[SCHEDULER ITER18 LR1]: 440067 words processed Wed Nov 18 11:33:18 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.093470. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46175 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90663 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40464 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57649 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09385 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11886 clock time -[SCHEDULER ITER18 LR1]: 480051 words processed Wed Nov 18 11:33:30 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.094432. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46175 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90218 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39395 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57383 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09144 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11907 clock time -[SCHEDULER ITER18 LR1]: 520140 words processed Wed Nov 18 11:33:42 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.094612. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46238 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91751 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44953 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59597 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10255 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12002 clock time -[SCHEDULER ITER18 LR1]: 560132 words processed Wed Nov 18 11:33:54 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.094673. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46101 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89952 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38704 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57082 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09178 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11903 clock time -[SCHEDULER ITER18 LR1]: 600118 words processed Wed Nov 18 11:34:06 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.094054. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46056 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90180 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39211 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57329 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09092 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11912 clock time -[SCHEDULER ITER18 LR1]: 640090 words processed Wed Nov 18 11:34:18 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.093141. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46103 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90819 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41022 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57965 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09385 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11971 clock time -[SCHEDULER ITER18 LR1]: 680075 words processed Wed Nov 18 11:34:30 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.092928. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46534 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91453 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43017 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58310 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09851 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11998 clock time -[SCHEDULER ITER18 LR1]: 720043 words processed Wed Nov 18 11:34:42 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.093002. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46267 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91386 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42703 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58369 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09721 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11979 clock time -[SCHEDULER ITER18 LR1]: 760012 words processed Wed Nov 18 11:34:54 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.092652. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95422 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51660 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60448 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10618 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12299 clock time -[SCHEDULER ITER18 LR1]: 800113 words processed Wed Nov 18 11:35:06 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.093194. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47398 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96804 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.55973 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62116 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11513 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12240 clock time -[SCHEDULER ITER18 LR1]: 840089 words processed Wed Nov 18 11:35:18 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.092664. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46658 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93123 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45958 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09883 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12123 clock time -[SCHEDULER ITER18 LR1]: 880052 words processed Wed Nov 18 11:35:30 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.092839. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46218 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91158 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41106 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57730 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09176 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11922 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:17:55 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:17:55 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:17:55 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:17:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:17:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:17:55 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:17:55 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER18 LR1]: 40092 words processed Wed Dec 23 17:18:11 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.095624. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48252 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96297 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75645 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77717 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11379 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14842 clock time +[SCHEDULER ITER18 LR1]: 80099 words processed Wed Dec 23 17:18:27 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.091822. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48081 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92841 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74896 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10078 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14816 clock time +[SCHEDULER ITER18 LR1]: 120004 words processed Wed Dec 23 17:18:43 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.087862. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47932 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92743 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67323 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74755 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10020 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14765 clock time +[SCHEDULER ITER18 LR1]: 160114 words processed Wed Dec 23 17:18:59 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.089453. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47893 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94947 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71756 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76593 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10632 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14805 clock time +[SCHEDULER ITER18 LR1]: 200066 words processed Wed Dec 23 17:19:15 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.089704. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93174 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67489 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75000 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10060 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14807 clock time +[SCHEDULER ITER18 LR1]: 240045 words processed Wed Dec 23 17:19:31 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.086063. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47425 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92766 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65872 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09667 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14751 clock time +[SCHEDULER ITER18 LR1]: 280057 words processed Wed Dec 23 17:19:47 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.082158. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47720 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96250 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74697 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77521 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10979 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14851 clock time +[SCHEDULER ITER18 LR1]: 320106 words processed Wed Dec 23 17:20:03 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.080544. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47488 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95132 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72174 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77016 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10478 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14821 clock time +[SCHEDULER ITER18 LR1]: 360024 words processed Wed Dec 23 17:20:19 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.080472. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47063 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92792 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66050 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74608 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09762 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14770 clock time +[SCHEDULER ITER18 LR1]: 400089 words processed Wed Dec 23 17:20:35 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.078570. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47219 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94787 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71077 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76726 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10568 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14793 clock time +[SCHEDULER ITER18 LR1]: 440067 words processed Wed Dec 23 17:20:51 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.079629. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47093 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92774 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65733 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74616 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09769 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14767 clock time +[SCHEDULER ITER18 LR1]: 480051 words processed Wed Dec 23 17:21:07 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.080532. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47164 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93378 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66822 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74900 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09951 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14771 clock time +[SCHEDULER ITER18 LR1]: 520140 words processed Wed Dec 23 17:21:23 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.081420. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47196 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94831 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71085 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76647 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10575 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14794 clock time +[SCHEDULER ITER18 LR1]: 560132 words processed Wed Dec 23 17:21:39 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.081725. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92676 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65455 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74699 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09684 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14732 clock time +[SCHEDULER ITER18 LR1]: 600118 words processed Wed Dec 23 17:21:55 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.079904. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47052 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92787 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65814 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74669 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09846 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14730 clock time +[SCHEDULER ITER18 LR1]: 640090 words processed Wed Dec 23 17:22:11 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.078754. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46955 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93027 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65917 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74861 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09606 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14749 clock time +[SCHEDULER ITER18 LR1]: 680075 words processed Wed Dec 23 17:22:27 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.078407. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46911 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92742 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65364 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74582 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09766 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14733 clock time +[SCHEDULER ITER18 LR1]: 720043 words processed Wed Dec 23 17:22:43 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.078504. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46981 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92838 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65999 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74810 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09768 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14758 clock time +[SCHEDULER ITER18 LR1]: 760012 words processed Wed Dec 23 17:22:59 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.077738. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47528 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96892 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73754 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76796 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10333 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14971 clock time +[SCHEDULER ITER18 LR1]: 800113 words processed Wed Dec 23 17:23:15 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.078191. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47178 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94944 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71972 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77385 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10564 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14796 clock time +[SCHEDULER ITER18 LR1]: 840089 words processed Wed Dec 23 17:23:31 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.077618. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47000 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92776 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65596 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74674 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09681 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14737 clock time +[SCHEDULER ITER18 LR1]: 880052 words processed Wed Dec 23 17:23:47 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.078059. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47029 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92661 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65955 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74718 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09927 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14744 clock time [LOG]LMSeqReader: file expires, closing. +(17:23:50 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER18 LR1]: Displaying result: -[SCHEDULER ITER18 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 137.34114630761> <PPL_OOV 123.86098737885> <LOGP -1945568.9240026> -[SCHEDULER ITER18 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER18 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 132.50573738613> <PPL_OOV 119.74577417145> <LOGP -1931927.8174362> +[SCHEDULER ITER18 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER18 LR1]: shuffling training file ===PEEK ON TEST 18=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER18 LR1]: 40087 words processed Wed Nov 18 11:35:39 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.194785. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46991 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91814 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.61391 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11487 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:23:52 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:23:52 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:23:52 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:23:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:23:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:23:52 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:23:52 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER18 LR1]: 40087 words processed Wed Dec 23 17:24:00 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.185074. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49133 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12705 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.88058 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14416 clock time [LOG]LMSeqReader: file expires, closing. +(17:24:08 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER18 LR1]: Displaying result: -[SCHEDULER ITER18 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 179.81187100755> <PPL_OOV 156.95421884454> <LOGP -180997.56787842> +[SCHEDULER ITER18 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 174.41683013471> <PPL_OOV 152.70857017295> <LOGP -180015.86095426> [SCHEDULER ITER18 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 18=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER18 LR1]: 40095 words processed Wed Nov 18 11:35:48 2015. - [SCHEDULER ITER18 LR1]: log prob per sample :-2.259428. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47031 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92876 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62514 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11466 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:24:08 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:24:08 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:24:08 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:24:08 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:24:08 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:24:08 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:24:08 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER18 LR1]: 40095 words processed Wed Dec 23 17:24:16 2015. + [SCHEDULER ITER18 LR1]: log prob per sample :-2.252189. + [global_conf.timer]: time spent on tnn_beforeprocess:0.50103 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.25097 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:7.02001 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14697 clock time [LOG]LMSeqReader: file expires, closing. +(17:24:22 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER18 LR1]: Displaying result: -[SCHEDULER ITER18 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 184.49928420535> <PPL_OOV 165.10810144726> <LOGP -163582.59597025> +[SCHEDULER ITER18 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 180.81153131269> <PPL_OOV 162.02354967405> <LOGP -162978.4838004> [SCHEDULER ITER18 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER18 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.18... +[SCHEDULER ITER18 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.18... [SCHEDULER ITER19 LR1]: preparing parameters... -[SCHEDULER ITER19 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.18... +[SCHEDULER ITER19 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.18... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34538074 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3667 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34541731 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120023 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69220662 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1186766 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69336928 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35890727 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70404066 +reading chunk 5 from 70432132 [SCHEDULER ITER19 LR1]: preparing parameters end. [SCHEDULER ITER19 LR1]: preparing layers... -(11:35:57 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:35:57 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:35:57 2015-11-18)[nerv] info: create layer: combinerL1 -(11:35:57 2015-11-18)[nerv] info: create layer: outputL -(11:35:57 2015-11-18)[nerv] info: create layer: softmaxL -(11:35:57 2015-11-18)[nerv] info: create layer: selectL1 +(17:24:28 2015-12-23)[nerv] info: create layer: sigmoidL1 +(17:24:28 2015-12-23)[nerv] info: create layer: combinerL1 +(17:24:28 2015-12-23)[nerv] info: create layer: selectL1 +(17:24:28 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(17:24:28 2015-12-23)[nerv] info: create layer: outputL +(17:24:28 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(17:24:28 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(17:24:28 2015-12-23)[nerv] info: create layer: softmaxL +(17:24:28 2015-12-23)[nerv] info: create layer: recurrentL1 +(17:24:28 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(17:24:28 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER19 LR1]: preparing layers end. [SCHEDULER ITER19 LR1]: Generate and initing TNN ... +(17:24:28 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -4858,253 +5517,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(17:24:28 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:24:28 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:24:28 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:24:28 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:24:28 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:24:28 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER19 LR1]: Initing TNN end. ===ITERATION 19 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER19 LR1]: 40092 words processed Wed Nov 18 11:36:08 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.090996. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46988 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91173 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47086 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60712 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11041 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12098 clock time -[SCHEDULER ITER19 LR1]: 80099 words processed Wed Nov 18 11:36:20 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.092216. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46934 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89179 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39641 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57506 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09537 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11965 clock time -[SCHEDULER ITER19 LR1]: 120004 words processed Wed Nov 18 11:36:32 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.089189. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46715 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89033 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39664 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57401 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09621 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11983 clock time -[SCHEDULER ITER19 LR1]: 160114 words processed Wed Nov 18 11:36:44 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.091547. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46744 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90322 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43612 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59313 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10464 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12014 clock time -[SCHEDULER ITER19 LR1]: 200066 words processed Wed Nov 18 11:36:56 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.092413. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46454 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89897 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40991 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57864 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09925 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12005 clock time -[SCHEDULER ITER19 LR1]: 240045 words processed Wed Nov 18 11:37:08 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.089188. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46391 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89287 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39182 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57381 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09516 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12003 clock time -[SCHEDULER ITER19 LR1]: 280057 words processed Wed Nov 18 11:37:20 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.085994. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46768 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91849 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47115 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60061 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10996 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12102 clock time -[SCHEDULER ITER19 LR1]: 320106 words processed Wed Nov 18 11:37:32 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.084639. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46345 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90954 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45099 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59862 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10691 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12048 clock time -[SCHEDULER ITER19 LR1]: 360024 words processed Wed Nov 18 11:37:44 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.083867. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46085 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89880 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41085 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58037 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10064 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11983 clock time -[SCHEDULER ITER19 LR1]: 400089 words processed Wed Nov 18 11:37:56 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.084462. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46410 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91425 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45488 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59876 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10614 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12118 clock time -[SCHEDULER ITER19 LR1]: 440067 words processed Wed Nov 18 11:38:08 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.085830. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46078 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89885 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39884 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57666 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09639 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11969 clock time -[SCHEDULER ITER19 LR1]: 480051 words processed Wed Nov 18 11:38:20 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.086788. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46045 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89161 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39250 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57527 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10092 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11994 clock time -[SCHEDULER ITER19 LR1]: 520140 words processed Wed Nov 18 11:38:32 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.087177. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46193 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90605 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43098 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59156 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10185 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12029 clock time -[SCHEDULER ITER19 LR1]: 560132 words processed Wed Nov 18 11:38:44 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.087613. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89090 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38672 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57348 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09837 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11986 clock time -[SCHEDULER ITER19 LR1]: 600118 words processed Wed Nov 18 11:38:56 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.085937. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45985 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89205 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38731 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57363 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09694 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11960 clock time -[SCHEDULER ITER19 LR1]: 640090 words processed Wed Nov 18 11:39:08 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.084914. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46037 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90001 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40597 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58015 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09800 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12059 clock time -[SCHEDULER ITER19 LR1]: 680075 words processed Wed Nov 18 11:39:20 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.084572. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46356 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91029 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43414 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58853 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10193 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12139 clock time -[SCHEDULER ITER19 LR1]: 720043 words processed Wed Nov 18 11:39:32 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.084724. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47208 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94667 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.52487 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60921 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11599 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12372 clock time -[SCHEDULER ITER19 LR1]: 760012 words processed Wed Nov 18 11:39:44 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.084702. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46438 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91615 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45742 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11009 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12183 clock time -[SCHEDULER ITER19 LR1]: 800113 words processed Wed Nov 18 11:39:56 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.085325. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46129 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90572 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43444 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59536 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10225 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12051 clock time -[SCHEDULER ITER19 LR1]: 840089 words processed Wed Nov 18 11:40:08 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.084822. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46052 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89527 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57762 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:24:28 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:24:28 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:24:28 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:24:28 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:24:28 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:24:28 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:24:28 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER19 LR1]: 40092 words processed Wed Dec 23 17:24:44 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.087723. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48206 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95425 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73484 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76712 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11058 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14831 clock time +[SCHEDULER ITER19 LR1]: 80099 words processed Wed Dec 23 17:25:00 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.083591. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48154 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93130 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67048 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74012 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10058 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14765 clock time +[SCHEDULER ITER19 LR1]: 120004 words processed Wed Dec 23 17:25:16 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.078821. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47875 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92805 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67203 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74397 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09991 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14773 clock time +[SCHEDULER ITER19 LR1]: 160114 words processed Wed Dec 23 17:25:32 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.080441. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47861 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94841 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71462 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76189 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10688 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14780 clock time +[SCHEDULER ITER19 LR1]: 200066 words processed Wed Dec 23 17:25:48 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.080848. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47445 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92621 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65637 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73995 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09781 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14767 clock time +[SCHEDULER ITER19 LR1]: 240045 words processed Wed Dec 23 17:26:04 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.077259. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47544 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93125 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66927 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74483 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09942 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11991 clock time -[SCHEDULER ITER19 LR1]: 880052 words processed Wed Nov 18 11:40:20 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.085047. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46152 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90107 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40960 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09522 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11999 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14783 clock time +[SCHEDULER ITER19 LR1]: 280057 words processed Wed Dec 23 17:26:20 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.073192. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47823 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97410 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76801 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77809 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11094 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14896 clock time +[SCHEDULER ITER19 LR1]: 320106 words processed Wed Dec 23 17:26:36 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.071725. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47376 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95676 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73433 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10887 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14898 clock time +[SCHEDULER ITER19 LR1]: 360024 words processed Wed Dec 23 17:26:52 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.071221. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47066 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92900 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66155 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74328 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09827 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14740 clock time +[SCHEDULER ITER19 LR1]: 400089 words processed Wed Dec 23 17:27:08 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.069357. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47374 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95197 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72458 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76808 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10872 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14867 clock time +[SCHEDULER ITER19 LR1]: 440067 words processed Wed Dec 23 17:27:24 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.070516. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46998 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92828 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74275 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09724 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14782 clock time +[SCHEDULER ITER19 LR1]: 480051 words processed Wed Dec 23 17:27:40 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.071463. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46945 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92716 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65207 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74187 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09778 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14758 clock time +[SCHEDULER ITER19 LR1]: 520140 words processed Wed Dec 23 17:27:56 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.071790. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47110 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95330 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71474 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76405 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10553 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14796 clock time +[SCHEDULER ITER19 LR1]: 560132 words processed Wed Dec 23 17:28:12 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.072127. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47184 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93191 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67259 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74925 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10137 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14792 clock time +[SCHEDULER ITER19 LR1]: 600118 words processed Wed Dec 23 17:28:28 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.071079. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46963 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93140 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66318 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74495 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09935 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14756 clock time +[SCHEDULER ITER19 LR1]: 640090 words processed Wed Dec 23 17:28:44 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.070203. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46965 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92820 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65488 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74226 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09784 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14733 clock time +[SCHEDULER ITER19 LR1]: 680075 words processed Wed Dec 23 17:29:00 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.069989. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46957 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93079 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66095 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74244 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10144 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14741 clock time +[SCHEDULER ITER19 LR1]: 720043 words processed Wed Dec 23 17:29:16 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.070078. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46996 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92879 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65998 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74365 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09901 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14734 clock time +[SCHEDULER ITER19 LR1]: 760012 words processed Wed Dec 23 17:29:32 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.069459. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46970 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93139 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66352 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74474 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09862 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14768 clock time +[SCHEDULER ITER19 LR1]: 800113 words processed Wed Dec 23 17:29:48 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.069868. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47199 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95421 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72718 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77085 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10772 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14855 clock time +[SCHEDULER ITER19 LR1]: 840089 words processed Wed Dec 23 17:30:04 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.069239. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47030 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94924 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69474 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75407 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10083 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14858 clock time +[SCHEDULER ITER19 LR1]: 880052 words processed Wed Dec 23 17:30:20 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.069622. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47524 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97014 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73818 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76133 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10654 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14956 clock time [LOG]LMSeqReader: file expires, closing. +(17:30:23 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER19 LR1]: Displaying result: -[SCHEDULER ITER19 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 134.79431248528> <PPL_OOV 121.66561590739> <LOGP -1938349.1048702> -[SCHEDULER ITER19 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER19 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 129.82741698799> <PPL_OOV 117.43351826521> <LOGP -1924055.9427717> +[SCHEDULER ITER19 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER19 LR1]: shuffling training file ===PEEK ON TEST 19=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER19 LR1]: 40087 words processed Wed Nov 18 11:40:29 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.194895. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47073 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91618 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.61384 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11614 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:30:25 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:30:25 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:30:25 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:30:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:30:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:30:25 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:30:25 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER19 LR1]: 40087 words processed Wed Dec 23 17:30:33 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.183430. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49229 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.11453 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.86911 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14335 clock time [LOG]LMSeqReader: file expires, closing. +(17:30:41 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER19 LR1]: Displaying result: -[SCHEDULER ITER19 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 179.35664560354> <PPL_OOV 156.98822353866> <LOGP -181005.32299711> +[SCHEDULER ITER19 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 173.60726128093> <PPL_OOV 152.10615717036> <LOGP -179874.36026251> [SCHEDULER ITER19 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 19=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER19 LR1]: 40095 words processed Wed Nov 18 11:40:38 2015. - [SCHEDULER ITER19 LR1]: log prob per sample :-2.258736. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46758 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90503 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.59738 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11484 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:30:41 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:30:41 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:30:41 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:30:41 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:30:41 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:30:41 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:30:41 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER19 LR1]: 40095 words processed Wed Dec 23 17:30:49 2015. + [SCHEDULER ITER19 LR1]: log prob per sample :-2.249865. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49094 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17498 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92591 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14223 clock time [LOG]LMSeqReader: file expires, closing. +(17:30:55 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER19 LR1]: Displaying result: -[SCHEDULER ITER19 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 184.03899331812> <PPL_OOV 165.02753091> <LOGP -163566.96020714> +[SCHEDULER ITER19 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 179.66186599734> <PPL_OOV 161.056931574> <LOGP -162786.80207374> [SCHEDULER ITER19 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER19 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.19... +[SCHEDULER ITER19 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.19... [SCHEDULER ITER20 LR1]: preparing parameters... -[SCHEDULER ITER20 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.19... +[SCHEDULER ITER20 LR1]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.19... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 1066494 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3667 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 1070151 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 120013 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 35761293 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1186045 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 35877568 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 35902539 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70414635 +reading chunk 5 from 70443178 [SCHEDULER ITER20 LR1]: preparing parameters end. [SCHEDULER ITER20 LR1]: preparing layers... -(11:40:47 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:40:47 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:40:47 2015-11-18)[nerv] info: create layer: combinerL1 -(11:40:47 2015-11-18)[nerv] info: create layer: outputL -(11:40:47 2015-11-18)[nerv] info: create layer: softmaxL -(11:40:47 2015-11-18)[nerv] info: create layer: selectL1 +(17:31:01 2015-12-23)[nerv] info: create layer: sigmoidL1 +(17:31:01 2015-12-23)[nerv] info: create layer: combinerL1 +(17:31:01 2015-12-23)[nerv] info: create layer: selectL1 +(17:31:01 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(17:31:01 2015-12-23)[nerv] info: create layer: outputL +(17:31:01 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(17:31:01 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(17:31:01 2015-12-23)[nerv] info: create layer: softmaxL +(17:31:01 2015-12-23)[nerv] info: create layer: recurrentL1 +(17:31:01 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(17:31:01 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER ITER20 LR1]: preparing layers end. [SCHEDULER ITER20 LR1]: Generate and initing TNN ... +(17:31:01 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -5120,253 +5815,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(17:31:01 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:31:01 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:31:01 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:31:01 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:31:01 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:31:01 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER ITER20 LR1]: Initing TNN end. ===ITERATION 20 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER20 LR1]: 40092 words processed Wed Nov 18 11:40:58 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.085050. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47070 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93031 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48032 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60405 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10566 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11927 clock time -[SCHEDULER ITER20 LR1]: 80099 words processed Wed Nov 18 11:41:10 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.087234. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47026 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92536 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45432 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58958 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10038 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12033 clock time -[SCHEDULER ITER20 LR1]: 120004 words processed Wed Nov 18 11:41:22 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.083289. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91228 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42540 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58007 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09641 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11867 clock time -[SCHEDULER ITER20 LR1]: 160114 words processed Wed Nov 18 11:41:34 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.084575. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46825 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92296 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45552 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59643 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10095 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11891 clock time -[SCHEDULER ITER20 LR1]: 200066 words processed Wed Nov 18 11:41:46 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.085180. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46440 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91039 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41367 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58088 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09134 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11886 clock time -[SCHEDULER ITER20 LR1]: 240045 words processed Wed Nov 18 11:41:58 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.082257. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46458 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91007 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41723 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57974 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09768 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11843 clock time -[SCHEDULER ITER20 LR1]: 280057 words processed Wed Nov 18 11:42:10 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.079229. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46649 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92216 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46209 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59853 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10453 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11882 clock time -[SCHEDULER ITER20 LR1]: 320106 words processed Wed Nov 18 11:42:22 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.077909. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46744 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93629 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.49664 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10926 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12071 clock time -[SCHEDULER ITER20 LR1]: 360024 words processed Wed Nov 18 11:42:34 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.077304. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46022 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91555 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42326 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09886 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11911 clock time -[SCHEDULER ITER20 LR1]: 400089 words processed Wed Nov 18 11:42:46 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.076911. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46210 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92434 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45640 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59622 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10538 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11932 clock time -[SCHEDULER ITER20 LR1]: 440067 words processed Wed Nov 18 11:42:58 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.078260. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46137 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91551 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58008 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09559 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11879 clock time -[SCHEDULER ITER20 LR1]: 480051 words processed Wed Nov 18 11:43:10 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.079301. - [global_conf.timer]: time spent on tnn_beforeprocess:0.45994 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91479 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41953 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58121 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09827 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11912 clock time -[SCHEDULER ITER20 LR1]: 520140 words processed Wed Nov 18 11:43:22 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.079337. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46392 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93347 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60159 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10597 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11999 clock time -[SCHEDULER ITER20 LR1]: 560132 words processed Wed Nov 18 11:43:34 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.079429. - [global_conf.timer]: time spent on tnn_beforeprocess:0.49459 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.10094 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.87257 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.70915 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14386 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13224 clock time -[SCHEDULER ITER20 LR1]: 600118 words processed Wed Nov 18 11:43:46 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.078745. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46556 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92739 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45134 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58882 clock time +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:31:01 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:31:01 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:31:01 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:31:01 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:31:01 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:31:01 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:31:01 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER20 LR1]: 40092 words processed Wed Dec 23 17:31:17 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.080297. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48086 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94919 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72543 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76921 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11140 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14788 clock time +[SCHEDULER ITER20 LR1]: 80099 words processed Wed Dec 23 17:31:33 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.076887. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48020 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92883 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66281 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74530 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09826 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14700 clock time +[SCHEDULER ITER20 LR1]: 120004 words processed Wed Dec 23 17:31:49 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.071736. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47830 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92623 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66282 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74505 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09884 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12006 clock time -[SCHEDULER ITER20 LR1]: 640090 words processed Wed Nov 18 11:43:58 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.077633. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46354 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92397 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45481 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59169 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10248 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12081 clock time -[SCHEDULER ITER20 LR1]: 680075 words processed Wed Nov 18 11:44:10 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.077152. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46260 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92093 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44081 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58723 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10110 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12010 clock time -[SCHEDULER ITER20 LR1]: 720043 words processed Wed Nov 18 11:44:22 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.077321. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46502 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93127 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45403 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58804 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09637 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12108 clock time -[SCHEDULER ITER20 LR1]: 760012 words processed Wed Nov 18 11:44:34 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.077145. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46358 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93263 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45589 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58963 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09891 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12033 clock time -[SCHEDULER ITER20 LR1]: 800113 words processed Wed Nov 18 11:44:46 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.077730. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47134 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96106 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.53236 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61040 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11125 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12156 clock time -[SCHEDULER ITER20 LR1]: 840089 words processed Wed Nov 18 11:44:58 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.077224. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46215 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91634 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43757 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58801 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09929 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12108 clock time -[SCHEDULER ITER20 LR1]: 880052 words processed Wed Nov 18 11:45:10 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.077427. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46151 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91869 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43426 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58722 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09678 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11982 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14701 clock time +[SCHEDULER ITER20 LR1]: 160114 words processed Wed Dec 23 17:32:05 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.072590. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47822 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94883 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72425 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77283 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11026 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14753 clock time +[SCHEDULER ITER20 LR1]: 200066 words processed Wed Dec 23 17:32:21 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.073557. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47473 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92564 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65516 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74578 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09697 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14693 clock time +[SCHEDULER ITER20 LR1]: 240045 words processed Wed Dec 23 17:32:37 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.070132. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47414 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92633 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65589 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74681 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09698 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time +[SCHEDULER ITER20 LR1]: 280057 words processed Wed Dec 23 17:32:53 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.065906. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47612 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94668 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71245 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76728 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10651 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14715 clock time +[SCHEDULER ITER20 LR1]: 320106 words processed Wed Dec 23 17:33:09 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.064432. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47501 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95038 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71696 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76980 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10512 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14764 clock time +[SCHEDULER ITER20 LR1]: 360024 words processed Wed Dec 23 17:33:25 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.063924. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46994 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92750 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74873 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09750 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14698 clock time +[SCHEDULER ITER20 LR1]: 400089 words processed Wed Dec 23 17:33:41 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.062142. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47182 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94737 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70687 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76729 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10617 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14736 clock time +[SCHEDULER ITER20 LR1]: 440067 words processed Wed Dec 23 17:33:57 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.063229. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46969 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92538 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65029 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74588 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09840 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14653 clock time +[SCHEDULER ITER20 LR1]: 480051 words processed Wed Dec 23 17:34:13 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.064257. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47126 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93227 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66815 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75239 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10043 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14725 clock time +[SCHEDULER ITER20 LR1]: 520140 words processed Wed Dec 23 17:34:29 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.064671. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47146 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94748 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70477 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76594 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10451 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14756 clock time +[SCHEDULER ITER20 LR1]: 560132 words processed Wed Dec 23 17:34:45 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.065018. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47229 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94100 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68446 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75696 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10213 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14754 clock time +[SCHEDULER ITER20 LR1]: 600118 words processed Wed Dec 23 17:35:01 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.063354. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46978 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92793 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65502 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74674 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09940 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14675 clock time +[SCHEDULER ITER20 LR1]: 640090 words processed Wed Dec 23 17:35:17 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.062165. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46938 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92811 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65405 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09982 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14678 clock time +[SCHEDULER ITER20 LR1]: 680075 words processed Wed Dec 23 17:35:33 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.061834. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46889 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92600 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65049 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74623 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09854 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14745 clock time +[SCHEDULER ITER20 LR1]: 720043 words processed Wed Dec 23 17:35:49 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.061880. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92717 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65457 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74745 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09880 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14694 clock time +[SCHEDULER ITER20 LR1]: 760012 words processed Wed Dec 23 17:36:05 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.061123. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47104 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92777 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66006 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09970 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14713 clock time +[SCHEDULER ITER20 LR1]: 800113 words processed Wed Dec 23 17:36:21 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.061554. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47080 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94646 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70343 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76569 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10600 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14727 clock time +[SCHEDULER ITER20 LR1]: 840089 words processed Wed Dec 23 17:36:37 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.061294. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93337 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67299 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75477 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10108 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14784 clock time +[SCHEDULER ITER20 LR1]: 880052 words processed Wed Dec 23 17:36:53 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.061688. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47015 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92691 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65782 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74843 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09949 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14681 clock time [LOG]LMSeqReader: file expires, closing. +(17:36:56 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER20 LR1]: Displaying result: -[SCHEDULER ITER20 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 132.32514776759> <PPL_OOV 119.54265480169> <LOGP -1931242.4317517> -[SCHEDULER ITER20 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER20 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 127.36605432463> <PPL_OOV 115.30466472085> <LOGP -1916670.1768219> +[SCHEDULER ITER20 LR1]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. [SCHEDULER ITER20 LR1]: shuffling training file ===PEEK ON TEST 20=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER20 LR1]: 40087 words processed Wed Nov 18 11:45:19 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.192376. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47284 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94029 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.64178 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11641 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:36:58 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:36:58 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:36:58 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:36:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:36:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:36:58 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:36:58 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER20 LR1]: 40087 words processed Wed Dec 23 17:37:06 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.182934. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49249 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12581 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.87950 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14411 clock time [LOG]LMSeqReader: file expires, closing. +(17:37:14 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER20 LR1]: Displaying result: -[SCHEDULER ITER20 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 178.8707175969> <PPL_OOV 156.1937916313> <LOGP -180823.70436399> +[SCHEDULER ITER20 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 172.99668858514> <PPL_OOV 152.14504736318> <LOGP -179883.51208102> [SCHEDULER ITER20 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 20=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER20 LR1]: 40095 words processed Wed Nov 18 11:45:29 2015. - [SCHEDULER ITER20 LR1]: log prob per sample :-2.256632. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47961 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99205 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.70687 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11972 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:37:14 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:37:14 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:37:14 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:37:14 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:37:14 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:37:14 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:37:14 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER20 LR1]: 40095 words processed Wed Dec 23 17:37:22 2015. + [SCHEDULER ITER20 LR1]: log prob per sample :-2.251030. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49231 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17524 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92700 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14299 clock time [LOG]LMSeqReader: file expires, closing. +(17:37:28 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER ITER20 LR1]: Displaying result: -[SCHEDULER ITER20 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 183.50506098011> <PPL_OOV 164.22139665816> <LOGP -163410.09772312> +[SCHEDULER ITER20 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 179.07618434179> <PPL_OOV 161.02236103576> <LOGP -162779.92539787> [SCHEDULER ITER20 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER20 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.20... +[SCHEDULER ITER20 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.20... -[SCHEDULER ITER21 LR1]: preparing parameters... -[SCHEDULER ITER21 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.20... +[SCHEDULER ITER21 LR0.6]: preparing parameters... +[SCHEDULER ITER21 LR0.6]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.20... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 1 from 34535500 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 34539157 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 69243371 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 69359649 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 5 from 70425551 -[SCHEDULER ITER21 LR1]: preparing parameters end. -[SCHEDULER ITER21 LR1]: preparing layers... -(11:45:38 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:45:38 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:45:38 2015-11-18)[nerv] info: create layer: combinerL1 -(11:45:38 2015-11-18)[nerv] info: create layer: outputL -(11:45:38 2015-11-18)[nerv] info: create layer: softmaxL -(11:45:38 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER21 LR1]: preparing layers end. -[SCHEDULER ITER21 LR1]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3667 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 120003 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1185289 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35914397 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70454283 +[SCHEDULER ITER21 LR0.6]: preparing parameters end. +[SCHEDULER ITER21 LR0.6]: preparing layers... +(17:37:34 2015-12-23)[nerv] info: create layer: sigmoidL1 +(17:37:34 2015-12-23)[nerv] info: create layer: combinerL1 +(17:37:34 2015-12-23)[nerv] info: create layer: selectL1 +(17:37:34 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(17:37:34 2015-12-23)[nerv] info: create layer: outputL +(17:37:34 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(17:37:34 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(17:37:34 2015-12-23)[nerv] info: create layer: softmaxL +(17:37:34 2015-12-23)[nerv] info: create layer: recurrentL1 +(17:37:34 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(17:37:34 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER21 LR0.6]: preparing layers end. +[SCHEDULER ITER21 LR0.6]: Generate and initing TNN ... +(17:37:34 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -5382,253 +6113,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER21 LR1]: Initing TNN end. -===ITERATION 21 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER21 LR1]: 40092 words processed Wed Nov 18 11:45:50 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.077161. - [global_conf.timer]: time spent on tnn_beforeprocess:0.50699 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.08887 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.91251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.73291 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.15631 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13152 clock time -[SCHEDULER ITER21 LR1]: 80099 words processed Wed Nov 18 11:46:02 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.078484. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48771 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98600 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62440 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.63444 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12169 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12598 clock time -[SCHEDULER ITER21 LR1]: 120004 words processed Wed Nov 18 11:46:14 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.074678. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47229 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93526 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48619 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59706 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10419 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12145 clock time -[SCHEDULER ITER21 LR1]: 160114 words processed Wed Nov 18 11:46:26 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.079716. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48247 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98340 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.61967 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64064 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12428 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12397 clock time -[SCHEDULER ITER21 LR1]: 200066 words processed Wed Nov 18 11:46:38 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.080377. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47351 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95018 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.52890 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60967 clock time +(17:37:34 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:37:34 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:37:34 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:37:34 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:37:34 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:37:34 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER21 LR0.6]: Initing TNN end. +===ITERATION 21 LR 0.600000=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:37:34 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:37:34 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:37:34 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:37:34 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:37:34 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:37:34 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:37:34 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER21 LR0.6]: 40092 words processed Wed Dec 23 17:37:50 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.040197. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48302 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95418 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73028 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76258 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11111 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12296 clock time -[SCHEDULER ITER21 LR1]: 240045 words processed Wed Nov 18 11:46:50 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.077124. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47907 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98218 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.59569 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12385 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12512 clock time -[SCHEDULER ITER21 LR1]: 280057 words processed Wed Nov 18 11:47:02 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.073853. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47034 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94403 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60744 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11559 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12177 clock time -[SCHEDULER ITER21 LR1]: 320106 words processed Wed Nov 18 11:47:14 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.072403. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46613 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92557 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47272 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59869 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11046 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12044 clock time -[SCHEDULER ITER21 LR1]: 360024 words processed Wed Nov 18 11:47:26 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.071235. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46305 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91240 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42633 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58025 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10088 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12000 clock time -[SCHEDULER ITER21 LR1]: 400089 words processed Wed Nov 18 11:47:38 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.069826. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46598 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92575 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47210 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59917 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11134 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12035 clock time -[SCHEDULER ITER21 LR1]: 440067 words processed Wed Nov 18 11:47:50 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.070909. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46525 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92679 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46226 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59104 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10695 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12148 clock time -[SCHEDULER ITER21 LR1]: 480051 words processed Wed Nov 18 11:48:02 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.071891. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46284 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91525 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42581 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58075 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10126 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11978 clock time -[SCHEDULER ITER21 LR1]: 520140 words processed Wed Nov 18 11:48:14 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.072260. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46420 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92169 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45230 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59549 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10151 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12023 clock time -[SCHEDULER ITER21 LR1]: 560132 words processed Wed Nov 18 11:48:26 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.072402. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46171 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91323 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41795 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57890 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09885 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12001 clock time -[SCHEDULER ITER21 LR1]: 600118 words processed Wed Nov 18 11:48:38 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.070887. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46130 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91167 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42037 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57916 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10251 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12002 clock time -[SCHEDULER ITER21 LR1]: 640090 words processed Wed Nov 18 11:48:50 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.070046. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46180 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91103 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42259 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58164 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10198 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11985 clock time -[SCHEDULER ITER21 LR1]: 680075 words processed Wed Nov 18 11:49:02 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.069803. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46227 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90922 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41674 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57714 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10320 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11970 clock time -[SCHEDULER ITER21 LR1]: 720043 words processed Wed Nov 18 11:49:14 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.069931. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46312 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91349 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42986 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58059 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10467 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11993 clock time -[SCHEDULER ITER21 LR1]: 760012 words processed Wed Nov 18 11:49:26 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.069535. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46294 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91337 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42327 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57970 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09977 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11953 clock time -[SCHEDULER ITER21 LR1]: 800113 words processed Wed Nov 18 11:49:38 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.070118. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46398 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92932 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47662 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59818 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11438 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12040 clock time -[SCHEDULER ITER21 LR1]: 840089 words processed Wed Nov 18 11:49:50 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.069680. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46273 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91298 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41699 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57633 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09978 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11962 clock time -[SCHEDULER ITER21 LR1]: 880052 words processed Wed Nov 18 11:50:02 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.069878. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46142 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91035 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41895 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57709 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10317 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11987 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14729 clock time +[SCHEDULER ITER21 LR0.6]: 80099 words processed Wed Dec 23 17:38:06 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.035397. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48265 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92770 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66824 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74261 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09839 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14831 clock time +[SCHEDULER ITER21 LR0.6]: 120004 words processed Wed Dec 23 17:38:22 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.028655. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48012 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92476 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66259 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09914 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14680 clock time +[SCHEDULER ITER21 LR0.6]: 160114 words processed Wed Dec 23 17:38:38 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.028404. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48106 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94643 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71143 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75988 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10591 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14772 clock time +[SCHEDULER ITER21 LR0.6]: 200066 words processed Wed Dec 23 17:38:54 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.027485. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47576 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92527 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65250 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73803 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09697 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14688 clock time +[SCHEDULER ITER21 LR0.6]: 240045 words processed Wed Dec 23 17:39:10 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.023077. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47608 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92675 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65649 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73921 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09798 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14724 clock time +[SCHEDULER ITER21 LR0.6]: 280057 words processed Wed Dec 23 17:39:26 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.018355. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47792 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94686 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71142 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75993 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10551 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14783 clock time +[SCHEDULER ITER21 LR0.6]: 320106 words processed Wed Dec 23 17:39:42 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.016186. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47588 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95963 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73351 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76773 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10686 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14838 clock time +[SCHEDULER ITER21 LR0.6]: 360024 words processed Wed Dec 23 17:39:58 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.014719. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47280 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94704 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70037 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75294 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10477 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14769 clock time +[SCHEDULER ITER21 LR0.6]: 400089 words processed Wed Dec 23 17:40:14 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.012066. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47345 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94691 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70790 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76279 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10492 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14762 clock time +[SCHEDULER ITER21 LR0.6]: 440067 words processed Wed Dec 23 17:40:30 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.012685. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47156 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92479 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65046 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73968 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09862 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14684 clock time +[SCHEDULER ITER21 LR0.6]: 480051 words processed Wed Dec 23 17:40:46 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.013246. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47136 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92490 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64572 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73922 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09609 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14664 clock time +[SCHEDULER ITER21 LR0.6]: 520140 words processed Wed Dec 23 17:41:02 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.012896. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47313 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95083 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70982 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76055 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10493 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14762 clock time +[SCHEDULER ITER21 LR0.6]: 560132 words processed Wed Dec 23 17:41:18 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.012742. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47235 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92717 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65415 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74105 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09844 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14693 clock time +[SCHEDULER ITER21 LR0.6]: 600118 words processed Wed Dec 23 17:41:34 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.010677. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92568 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65029 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74019 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09696 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14725 clock time +[SCHEDULER ITER21 LR0.6]: 640090 words processed Wed Dec 23 17:41:50 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.008913. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47013 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92411 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64620 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73962 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09657 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14703 clock time +[SCHEDULER ITER21 LR0.6]: 680075 words processed Wed Dec 23 17:42:06 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.008153. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47067 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92597 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64920 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73961 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09736 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14736 clock time +[SCHEDULER ITER21 LR0.6]: 720043 words processed Wed Dec 23 17:42:22 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.007582. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92503 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64886 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73979 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09654 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14685 clock time +[SCHEDULER ITER21 LR0.6]: 760012 words processed Wed Dec 23 17:42:38 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.006243. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47097 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92547 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65048 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73922 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09793 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14677 clock time +[SCHEDULER ITER21 LR0.6]: 800113 words processed Wed Dec 23 17:42:54 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.006301. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47254 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94595 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70289 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75881 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10646 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14738 clock time +[SCHEDULER ITER21 LR0.6]: 840089 words processed Wed Dec 23 17:43:10 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.005456. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92674 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65456 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74324 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09659 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14742 clock time +[SCHEDULER ITER21 LR0.6]: 880052 words processed Wed Dec 23 17:43:26 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.005475. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92584 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65132 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73884 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09822 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14681 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER21 LR1]: Displaying result: -[SCHEDULER ITER21 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 129.94447642588> <PPL_OOV 117.48605218627> <LOGP -1924236.5045858> -[SCHEDULER ITER21 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER21 LR1]: shuffling training file +(17:43:29 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER21 LR0.6]: Displaying result: +[SCHEDULER ITER21 LR0.6]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 111.94417036363> <PPL_OOV 101.28916437355> <LOGP -1864349.2926916> +[SCHEDULER ITER21 LR0.6]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER21 LR0.6]: shuffling training file ===PEEK ON TEST 21=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER21 LR1]: 40087 words processed Wed Nov 18 11:50:11 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.191794. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47015 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93243 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62877 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11544 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:43:31 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:43:31 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:43:31 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:43:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:43:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:43:31 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:43:31 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER21 LR0.6]: 40087 words processed Wed Dec 23 17:43:39 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.160415. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49577 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.15825 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92041 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14603 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER21 LR1]: Displaying result: -[SCHEDULER ITER21 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 177.54148794518> <PPL_OOV 155.83031275648> <LOGP -180740.29960477> -[SCHEDULER ITER21 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(17:43:47 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER21 LR0.6]: Displaying result: +[SCHEDULER ITER21 LR0.6]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 165.06493538423> <PPL_OOV 144.47099950033> <LOGP -178030.72123784> +[SCHEDULER ITER21 LR0.6]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 21=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER21 LR1]: 40095 words processed Wed Nov 18 11:50:20 2015. - [SCHEDULER ITER21 LR1]: log prob per sample :-2.256591. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47050 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92842 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62402 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11516 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:43:47 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:43:47 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:43:47 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:43:47 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:43:47 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:43:47 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:43:47 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER21 LR0.6]: 40095 words processed Wed Dec 23 17:43:55 2015. + [SCHEDULER ITER21 LR0.6]: log prob per sample :-2.229598. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49213 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.16408 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.91729 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14283 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER21 LR1]: Displaying result: -[SCHEDULER ITER21 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 182.38397970157> <PPL_OOV 163.87744761893> <LOGP -163342.9355447> -[SCHEDULER ITER21 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER21 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.21... - -[SCHEDULER ITER22 LR1]: preparing parameters... -[SCHEDULER ITER22 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.21... +(17:44:01 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER21 LR0.6]: Displaying result: +[SCHEDULER ITER21 LR0.6]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 170.47629110514> <PPL_OOV 152.72717948356> <LOGP -161085.66860556> +[SCHEDULER ITER21 LR0.6]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER21 LR0.6]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.21... + +[SCHEDULER ITER22 LR0.36]: preparing parameters... +[SCHEDULER ITER22 LR0.36]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.21... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 1 from 1065368 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 1069025 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 35788267 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 35904550 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 5 from 70438596 -[SCHEDULER ITER22 LR1]: preparing parameters end. -[SCHEDULER ITER22 LR1]: preparing layers... -(11:50:29 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:50:29 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:50:29 2015-11-18)[nerv] info: create layer: combinerL1 -(11:50:29 2015-11-18)[nerv] info: create layer: outputL -(11:50:29 2015-11-18)[nerv] info: create layer: softmaxL -(11:50:29 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER22 LR1]: preparing layers end. -[SCHEDULER ITER22 LR1]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3667 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 119974 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1184623 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35918345 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70457295 +[SCHEDULER ITER22 LR0.36]: preparing parameters end. +[SCHEDULER ITER22 LR0.36]: preparing layers... +(17:44:07 2015-12-23)[nerv] info: create layer: sigmoidL1 +(17:44:07 2015-12-23)[nerv] info: create layer: combinerL1 +(17:44:07 2015-12-23)[nerv] info: create layer: selectL1 +(17:44:07 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(17:44:07 2015-12-23)[nerv] info: create layer: outputL +(17:44:07 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(17:44:07 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(17:44:07 2015-12-23)[nerv] info: create layer: softmaxL +(17:44:07 2015-12-23)[nerv] info: create layer: recurrentL1 +(17:44:07 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(17:44:07 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER22 LR0.36]: preparing layers end. +[SCHEDULER ITER22 LR0.36]: Generate and initing TNN ... +(17:44:07 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -5644,253 +6411,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER22 LR1]: Initing TNN end. -===ITERATION 22 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER22 LR1]: 40092 words processed Wed Nov 18 11:50:40 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.070689. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47294 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92936 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60171 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11017 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12013 clock time -[SCHEDULER ITER22 LR1]: 80099 words processed Wed Nov 18 11:50:52 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.070559. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47320 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91899 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44703 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58440 clock time +(17:44:07 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:44:07 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:44:07 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:44:07 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:44:07 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:44:07 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER22 LR0.36]: Initing TNN end. +===ITERATION 22 LR 0.360000=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:44:07 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:44:07 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:44:07 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:44:07 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:44:07 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:44:07 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:44:07 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER22 LR0.36]: 40092 words processed Wed Dec 23 17:44:23 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-2.005873. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48091 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95221 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73664 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77069 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11431 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14786 clock time +[SCHEDULER ITER22 LR0.36]: 80099 words processed Wed Dec 23 17:44:39 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-2.000565. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92935 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67252 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74860 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09959 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14775 clock time +[SCHEDULER ITER22 LR0.36]: 120004 words processed Wed Dec 23 17:44:55 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.994420. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47853 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92953 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67569 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74859 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10071 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14706 clock time +[SCHEDULER ITER22 LR0.36]: 160114 words processed Wed Dec 23 17:45:11 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.993940. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48144 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97562 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76864 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77984 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11072 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14929 clock time +[SCHEDULER ITER22 LR0.36]: 200066 words processed Wed Dec 23 17:45:27 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.992993. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47457 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93004 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66532 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74651 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09850 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14724 clock time +[SCHEDULER ITER22 LR0.36]: 240045 words processed Wed Dec 23 17:45:43 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.988768. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47486 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93015 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66494 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74713 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09768 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14735 clock time +[SCHEDULER ITER22 LR0.36]: 280057 words processed Wed Dec 23 17:45:59 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.984192. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47574 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95065 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72210 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76901 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10679 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14784 clock time +[SCHEDULER ITER22 LR0.36]: 320106 words processed Wed Dec 23 17:46:15 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.982082. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47355 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95035 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72459 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77230 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10808 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14818 clock time +[SCHEDULER ITER22 LR0.36]: 360024 words processed Wed Dec 23 17:46:31 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.980631. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46970 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92904 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66998 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75270 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09998 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14747 clock time +[SCHEDULER ITER22 LR0.36]: 400089 words processed Wed Dec 23 17:46:47 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.977983. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47582 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97722 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78104 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10986 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14917 clock time +[SCHEDULER ITER22 LR0.36]: 440067 words processed Wed Dec 23 17:47:03 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.978590. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93212 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66801 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75087 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09860 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14771 clock time +[SCHEDULER ITER22 LR0.36]: 480051 words processed Wed Dec 23 17:47:19 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.979180. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47004 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92922 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66327 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09966 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14732 clock time +[SCHEDULER ITER22 LR0.36]: 520140 words processed Wed Dec 23 17:47:35 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.978771. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47370 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96643 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74751 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77755 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10854 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14874 clock time +[SCHEDULER ITER22 LR0.36]: 560132 words processed Wed Dec 23 17:47:51 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.978424. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46984 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93065 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66226 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74918 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09876 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14726 clock time +[SCHEDULER ITER22 LR0.36]: 600118 words processed Wed Dec 23 17:48:07 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.976363. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47048 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93572 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67625 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75516 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09934 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14756 clock time +[SCHEDULER ITER22 LR0.36]: 640090 words processed Wed Dec 23 17:48:23 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.974399. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92826 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66130 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74922 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09984 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14731 clock time +[SCHEDULER ITER22 LR0.36]: 680075 words processed Wed Dec 23 17:48:39 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.973441. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46887 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92888 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66127 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74886 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10062 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12067 clock time -[SCHEDULER ITER22 LR1]: 120004 words processed Wed Nov 18 11:51:04 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.067385. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47040 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91730 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43868 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58108 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09874 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11994 clock time -[SCHEDULER ITER22 LR1]: 160114 words processed Wed Nov 18 11:51:16 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.071316. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47112 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92989 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47913 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60026 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10819 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12014 clock time -[SCHEDULER ITER22 LR1]: 200066 words processed Wed Nov 18 11:51:28 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.072229. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46624 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91882 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58491 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09859 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11998 clock time -[SCHEDULER ITER22 LR1]: 240045 words processed Wed Nov 18 11:51:40 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.068766. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91796 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44076 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58383 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10261 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11978 clock time -[SCHEDULER ITER22 LR1]: 280057 words processed Wed Nov 18 11:51:52 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.065662. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46883 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93045 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48406 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60226 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10881 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12018 clock time -[SCHEDULER ITER22 LR1]: 320106 words processed Wed Nov 18 11:52:04 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.064301. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46617 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92558 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46902 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59649 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10918 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12003 clock time -[SCHEDULER ITER22 LR1]: 360024 words processed Wed Nov 18 11:52:16 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.063798. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46373 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92877 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45268 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58561 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10090 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12041 clock time -[SCHEDULER ITER22 LR1]: 400089 words processed Wed Nov 18 11:52:28 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.062554. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46397 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92860 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47055 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10731 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11995 clock time -[SCHEDULER ITER22 LR1]: 440067 words processed Wed Nov 18 11:52:40 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.063656. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46288 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91402 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42446 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57999 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10101 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11957 clock time -[SCHEDULER ITER22 LR1]: 480051 words processed Wed Nov 18 11:52:52 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.064683. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46269 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91547 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42449 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57975 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10091 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11943 clock time -[SCHEDULER ITER22 LR1]: 520140 words processed Wed Nov 18 11:53:04 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.064899. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46370 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92490 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46572 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59769 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10830 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12000 clock time -[SCHEDULER ITER22 LR1]: 560132 words processed Wed Nov 18 11:53:16 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.065037. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91854 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43594 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58490 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10161 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11978 clock time -[SCHEDULER ITER22 LR1]: 600118 words processed Wed Nov 18 11:53:28 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.063542. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91652 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42318 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58069 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09594 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11929 clock time -[SCHEDULER ITER22 LR1]: 640090 words processed Wed Nov 18 11:53:40 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.062551. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46485 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92311 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44782 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58853 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10220 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11985 clock time -[SCHEDULER ITER22 LR1]: 680075 words processed Wed Nov 18 11:53:52 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.062470. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46150 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91661 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42771 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58335 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09918 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11946 clock time -[SCHEDULER ITER22 LR1]: 720043 words processed Wed Nov 18 11:54:04 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.062579. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46247 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91963 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43752 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58354 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10265 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11980 clock time -[SCHEDULER ITER22 LR1]: 760012 words processed Wed Nov 18 11:54:16 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.062429. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46192 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91687 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42592 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58204 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09608 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11960 clock time -[SCHEDULER ITER22 LR1]: 800113 words processed Wed Nov 18 11:54:28 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.063025. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46595 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93408 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48883 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60521 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11036 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12087 clock time -[SCHEDULER ITER22 LR1]: 840089 words processed Wed Nov 18 11:54:40 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.062518. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92301 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43526 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58309 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09901 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11982 clock time -[SCHEDULER ITER22 LR1]: 880052 words processed Wed Nov 18 11:54:52 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.062751. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46309 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92228 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44665 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58868 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10188 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11987 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14749 clock time +[SCHEDULER ITER22 LR0.36]: 720043 words processed Wed Dec 23 17:48:55 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.972558. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46962 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93725 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75197 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10208 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14757 clock time +[SCHEDULER ITER22 LR0.36]: 760012 words processed Wed Dec 23 17:49:11 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.970934. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46982 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93231 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67046 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75116 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10092 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14712 clock time +[SCHEDULER ITER22 LR0.36]: 800113 words processed Wed Dec 23 17:49:27 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.970775. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47062 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95002 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71321 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76938 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10532 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14770 clock time +[SCHEDULER ITER22 LR0.36]: 840089 words processed Wed Dec 23 17:49:43 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.969749. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47008 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92947 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66240 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74951 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09869 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14737 clock time +[SCHEDULER ITER22 LR0.36]: 880052 words processed Wed Dec 23 17:49:59 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-1.969551. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47009 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94050 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69134 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76048 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10145 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14787 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER22 LR1]: Displaying result: -[SCHEDULER ITER22 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 127.72963687075> <PPL_OOV 115.57633745052> <LOGP -1917620.263724> -[SCHEDULER ITER22 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER22 LR1]: shuffling training file +(17:50:02 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER22 LR0.36]: Displaying result: +[SCHEDULER ITER22 LR0.36]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 102.90415532182> <PPL_OOV 93.239411487664> <LOGP -1830918.0515991> +[SCHEDULER ITER22 LR0.36]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER22 LR0.36]: shuffling training file ===PEEK ON TEST 22=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER22 LR1]: 40087 words processed Wed Nov 18 11:55:01 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.190838. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47028 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92927 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62624 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11563 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:50:04 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:50:04 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:50:04 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:50:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:50:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:50:04 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:50:04 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER22 LR0.36]: 40087 words processed Wed Dec 23 17:50:12 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-2.139596. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49356 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12570 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.88146 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14419 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER22 LR1]: Displaying result: -[SCHEDULER ITER22 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 177.63579916812> <PPL_OOV 155.49815507676> <LOGP -180663.9115879> -[SCHEDULER ITER22 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(17:50:20 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER22 LR0.36]: Displaying result: +[SCHEDULER ITER22 LR0.36]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 157.43242251224> <PPL_OOV 137.83477126722> <LOGP -176347.3450501> +[SCHEDULER ITER22 LR0.36]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 22=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER22 LR1]: 40095 words processed Wed Nov 18 11:55:10 2015. - [SCHEDULER ITER22 LR1]: log prob per sample :-2.256433. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47013 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92879 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62503 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11514 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:50:20 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:50:20 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:50:20 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:50:20 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:50:20 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:50:20 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:50:20 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER22 LR0.36]: 40095 words processed Wed Dec 23 17:50:28 2015. + [SCHEDULER ITER22 LR0.36]: log prob per sample :-2.210505. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49209 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17266 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92528 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14316 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER22 LR1]: Displaying result: -[SCHEDULER ITER22 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 182.55913997681> <PPL_OOV 163.69972113613> <LOGP -163308.17602403> -[SCHEDULER ITER22 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER22 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.22... - -[SCHEDULER ITER23 LR1]: preparing parameters... -[SCHEDULER ITER23 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.22... +(17:50:34 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER22 LR0.36]: Displaying result: +[SCHEDULER ITER22 LR0.36]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 162.48031080592> <PPL_OOV 145.62119251027> <LOGP -159559.44525848> +[SCHEDULER ITER22 LR0.36]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER22 LR0.36]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.22... + +[SCHEDULER ITER23 LR0.216]: preparing parameters... +[SCHEDULER ITER23 LR0.216]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.22... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 1 from 34532317 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 34535974 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 69270165 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 69386444 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 5 from 70451255 -[SCHEDULER ITER23 LR1]: preparing parameters end. -[SCHEDULER ITER23 LR1]: preparing layers... -(11:55:19 2015-11-18)[nerv] info: create layer: recurrentL1 -(11:55:19 2015-11-18)[nerv] info: create layer: sigmoidL1 -(11:55:19 2015-11-18)[nerv] info: create layer: combinerL1 -(11:55:19 2015-11-18)[nerv] info: create layer: outputL -(11:55:19 2015-11-18)[nerv] info: create layer: softmaxL -(11:55:19 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER23 LR1]: preparing layers end. -[SCHEDULER ITER23 LR1]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3666 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 119952 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1184089 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35919430 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70457399 +[SCHEDULER ITER23 LR0.216]: preparing parameters end. +[SCHEDULER ITER23 LR0.216]: preparing layers... +(17:50:40 2015-12-23)[nerv] info: create layer: sigmoidL1 +(17:50:40 2015-12-23)[nerv] info: create layer: combinerL1 +(17:50:40 2015-12-23)[nerv] info: create layer: selectL1 +(17:50:40 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(17:50:40 2015-12-23)[nerv] info: create layer: outputL +(17:50:40 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(17:50:40 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(17:50:40 2015-12-23)[nerv] info: create layer: softmaxL +(17:50:40 2015-12-23)[nerv] info: create layer: recurrentL1 +(17:50:40 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(17:50:40 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER23 LR0.216]: preparing layers end. +[SCHEDULER ITER23 LR0.216]: Generate and initing TNN ... +(17:50:40 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -5906,253 +6709,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER23 LR1]: Initing TNN end. -===ITERATION 23 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER23 LR1]: 40092 words processed Wed Nov 18 11:55:30 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.095018. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47532 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92781 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.49415 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61267 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10823 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12050 clock time -[SCHEDULER ITER23 LR1]: 80099 words processed Wed Nov 18 11:55:42 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.086515. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47435 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91415 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43939 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58751 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09727 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12031 clock time -[SCHEDULER ITER23 LR1]: 120004 words processed Wed Nov 18 11:55:54 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.077029. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47295 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91466 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44401 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58747 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09828 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12025 clock time -[SCHEDULER ITER23 LR1]: 160114 words processed Wed Nov 18 11:56:06 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.075436. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47346 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92640 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47844 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60507 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10443 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12070 clock time -[SCHEDULER ITER23 LR1]: 200066 words processed Wed Nov 18 11:56:18 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.073590. +(17:50:40 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:50:40 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:50:40 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:50:40 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:50:40 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:50:40 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER23 LR0.216]: Initing TNN end. +===ITERATION 23 LR 0.216000=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:50:40 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:50:40 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:50:40 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:50:40 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:50:40 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:50:40 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:50:40 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER23 LR0.216]: 40092 words processed Wed Dec 23 17:50:56 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.982744. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48231 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96719 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75231 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76849 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11569 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14684 clock time +[SCHEDULER ITER23 LR0.216]: 80099 words processed Wed Dec 23 17:51:12 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.977270. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48090 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93954 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68111 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74077 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10506 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14638 clock time +[SCHEDULER ITER23 LR0.216]: 120004 words processed Wed Dec 23 17:51:28 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.971308. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47789 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93547 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67746 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74232 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10289 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14681 clock time +[SCHEDULER ITER23 LR0.216]: 160114 words processed Wed Dec 23 17:51:44 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.970880. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48215 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98222 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.77114 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77223 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11361 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14834 clock time +[SCHEDULER ITER23 LR0.216]: 200066 words processed Wed Dec 23 17:52:00 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.969944. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47510 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93623 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67771 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74451 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10534 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14695 clock time +[SCHEDULER ITER23 LR0.216]: 240045 words processed Wed Dec 23 17:52:16 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.965862. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47486 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93843 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74245 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10333 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14701 clock time +[SCHEDULER ITER23 LR0.216]: 280057 words processed Wed Dec 23 17:52:32 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.961394. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47627 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95592 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72678 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76344 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11065 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14715 clock time +[SCHEDULER ITER23 LR0.216]: 320106 words processed Wed Dec 23 17:52:48 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.959243. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47399 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95546 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72134 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76366 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10851 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14703 clock time +[SCHEDULER ITER23 LR0.216]: 360024 words processed Wed Dec 23 17:53:04 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.957700. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47583 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97939 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76242 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10852 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14854 clock time +[SCHEDULER ITER23 LR0.216]: 400089 words processed Wed Dec 23 17:53:20 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.955047. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47207 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97217 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74700 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76956 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11163 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14794 clock time +[SCHEDULER ITER23 LR0.216]: 440067 words processed Wed Dec 23 17:53:36 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.955602. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93619 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66351 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74186 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10055 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14676 clock time +[SCHEDULER ITER23 LR0.216]: 480051 words processed Wed Dec 23 17:53:52 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.956153. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47303 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95905 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70348 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74944 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10468 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14779 clock time +[SCHEDULER ITER23 LR0.216]: 520140 words processed Wed Dec 23 17:54:08 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.955661. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47163 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95676 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72557 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76750 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10910 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14798 clock time +[SCHEDULER ITER23 LR0.216]: 560132 words processed Wed Dec 23 17:54:24 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.955237. [global_conf.timer]: time spent on tnn_beforeprocess:0.46989 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91459 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43968 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58856 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09874 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12004 clock time -[SCHEDULER ITER23 LR1]: 240045 words processed Wed Nov 18 11:56:30 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.069125. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46995 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91617 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43926 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09789 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11978 clock time -[SCHEDULER ITER23 LR1]: 280057 words processed Wed Nov 18 11:56:42 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.064858. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47028 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92587 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47623 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60479 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10325 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12053 clock time -[SCHEDULER ITER23 LR1]: 320106 words processed Wed Nov 18 11:56:54 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.062961. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46854 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92348 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47390 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60439 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10625 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12026 clock time -[SCHEDULER ITER23 LR1]: 360024 words processed Wed Nov 18 11:57:06 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.061620. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46561 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91583 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43891 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58795 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09882 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12039 clock time -[SCHEDULER ITER23 LR1]: 400089 words processed Wed Nov 18 11:57:18 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.059833. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46677 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92731 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47574 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60452 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10642 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12083 clock time -[SCHEDULER ITER23 LR1]: 440067 words processed Wed Nov 18 11:57:30 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.060592. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46511 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91411 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43198 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58752 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09873 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12000 clock time -[SCHEDULER ITER23 LR1]: 480051 words processed Wed Nov 18 11:57:42 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.061365. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46585 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91257 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43158 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58704 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10079 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12013 clock time -[SCHEDULER ITER23 LR1]: 520140 words processed Wed Nov 18 11:57:54 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.061413. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46678 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92455 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46962 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60354 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10406 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12079 clock time -[SCHEDULER ITER23 LR1]: 560132 words processed Wed Nov 18 11:58:06 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.061345. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46405 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90491 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42098 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58659 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09929 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11995 clock time -[SCHEDULER ITER23 LR1]: 600118 words processed Wed Nov 18 11:58:18 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.059732. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92628 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48292 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60492 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10576 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12264 clock time -[SCHEDULER ITER23 LR1]: 640090 words processed Wed Nov 18 11:58:30 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.058492. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46412 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91224 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45254 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59944 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10306 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12187 clock time -[SCHEDULER ITER23 LR1]: 680075 words processed Wed Nov 18 11:58:42 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.058583. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46580 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91796 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46151 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60252 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10256 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12175 clock time -[SCHEDULER ITER23 LR1]: 720043 words processed Wed Nov 18 11:58:54 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.058929. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46224 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89935 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40698 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58321 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09721 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11907 clock time -[SCHEDULER ITER23 LR1]: 760012 words processed Wed Nov 18 11:59:06 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.058524. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46265 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90315 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41084 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58366 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09579 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11908 clock time -[SCHEDULER ITER23 LR1]: 800113 words processed Wed Nov 18 11:59:18 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.059124. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46432 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91279 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44913 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60026 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10388 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11947 clock time -[SCHEDULER ITER23 LR1]: 840089 words processed Wed Nov 18 11:59:30 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.058808. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46269 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90330 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58365 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09636 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11883 clock time -[SCHEDULER ITER23 LR1]: 880052 words processed Wed Nov 18 11:59:42 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.059093. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46466 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90803 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42554 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58569 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10036 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93590 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66242 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74146 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10091 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14651 clock time +[SCHEDULER ITER23 LR0.216]: 600118 words processed Wed Dec 23 17:54:40 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.953096. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93542 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66200 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74077 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10061 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14653 clock time +[SCHEDULER ITER23 LR0.216]: 640090 words processed Wed Dec 23 17:54:56 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.951039. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46950 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93498 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66617 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74395 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10288 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14655 clock time +[SCHEDULER ITER23 LR0.216]: 680075 words processed Wed Dec 23 17:55:12 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.949967. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47057 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94053 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68168 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10518 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14733 clock time +[SCHEDULER ITER23 LR0.216]: 720043 words processed Wed Dec 23 17:55:28 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.948910. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47069 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94465 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68044 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74596 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10123 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14693 clock time +[SCHEDULER ITER23 LR0.216]: 760012 words processed Wed Dec 23 17:55:44 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.947129. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46976 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93552 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66260 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74072 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10007 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14653 clock time +[SCHEDULER ITER23 LR0.216]: 800113 words processed Wed Dec 23 17:56:00 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.946876. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47210 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95636 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72275 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76463 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11084 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14720 clock time +[SCHEDULER ITER23 LR0.216]: 840089 words processed Wed Dec 23 17:56:16 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.945744. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46909 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93812 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66884 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74374 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10248 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14650 clock time +[SCHEDULER ITER23 LR0.216]: 880052 words processed Wed Dec 23 17:56:32 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-1.945415. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47170 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96118 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70715 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75110 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10359 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14721 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER23 LR1]: Displaying result: -[SCHEDULER ITER23 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 126.5933315247> <PPL_OOV 114.59548203272> <LOGP -1914179.4532121> -[SCHEDULER ITER23 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER23 LR1]: shuffling training file +(17:56:35 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER23 LR0.216]: Displaying result: +[SCHEDULER ITER23 LR0.216]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 97.214031873589> <PPL_OOV 88.190059408003> <LOGP -1808440.6932284> +[SCHEDULER ITER23 LR0.216]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER23 LR0.216]: shuffling training file ===PEEK ON TEST 23=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER23 LR1]: 40087 words processed Wed Nov 18 11:59:51 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.187803. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47363 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93161 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.63241 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11638 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:56:37 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:56:37 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:56:37 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:56:37 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:56:37 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:56:37 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:56:37 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER23 LR0.216]: 40087 words processed Wed Dec 23 17:56:45 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-2.127647. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49450 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.15009 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.90886 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14538 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER23 LR1]: Displaying result: -[SCHEDULER ITER23 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 176.69113282554> <PPL_OOV 154.4522825706> <LOGP -180422.316696> -[SCHEDULER ITER23 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(17:56:53 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER23 LR0.216]: Displaying result: +[SCHEDULER ITER23 LR0.216]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 153.55002097359> <PPL_OOV 134.25500732518> <LOGP -175405.31050562> +[SCHEDULER ITER23 LR0.216]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 23=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER23 LR1]: 40095 words processed Wed Nov 18 12:00:00 2015. - [SCHEDULER ITER23 LR1]: log prob per sample :-2.253996. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47175 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92880 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62649 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11533 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:56:53 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:56:53 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:56:53 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:56:53 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:56:53 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:56:53 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:56:53 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER23 LR0.216]: 40095 words processed Wed Dec 23 17:57:01 2015. + [SCHEDULER ITER23 LR0.216]: log prob per sample :-2.199868. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49405 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17815 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.93243 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14260 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER23 LR1]: Displaying result: -[SCHEDULER ITER23 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 181.18925970679> <PPL_OOV 162.3281892853> <LOGP -163038.65732971> -[SCHEDULER ITER23 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER23 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.23... - -[SCHEDULER ITER24 LR1]: preparing parameters... -[SCHEDULER ITER24 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.23... +(17:57:07 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER23 LR0.216]: Displaying result: +[SCHEDULER ITER23 LR0.216]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 158.37946618855> <PPL_OOV 141.80412102265> <LOGP -158708.56974011> +[SCHEDULER ITER23 LR0.216]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER23 LR0.216]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.23... + +[SCHEDULER ITER24 LR0.1296]: preparing parameters... +[SCHEDULER ITER24 LR0.1296]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.23... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 1 from 1064210 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 1067867 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 35817408 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 35933682 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 5 from 70464168 -[SCHEDULER ITER24 LR1]: preparing parameters end. -[SCHEDULER ITER24 LR1]: preparing layers... -(12:00:09 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:00:09 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:00:09 2015-11-18)[nerv] info: create layer: combinerL1 -(12:00:09 2015-11-18)[nerv] info: create layer: outputL -(12:00:09 2015-11-18)[nerv] info: create layer: softmaxL -(12:00:09 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER24 LR1]: preparing layers end. -[SCHEDULER ITER24 LR1]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3665 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 119952 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1183741 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35919568 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70457171 +[SCHEDULER ITER24 LR0.1296]: preparing parameters end. +[SCHEDULER ITER24 LR0.1296]: preparing layers... +(17:57:13 2015-12-23)[nerv] info: create layer: sigmoidL1 +(17:57:13 2015-12-23)[nerv] info: create layer: combinerL1 +(17:57:13 2015-12-23)[nerv] info: create layer: selectL1 +(17:57:13 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(17:57:13 2015-12-23)[nerv] info: create layer: outputL +(17:57:13 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(17:57:13 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(17:57:13 2015-12-23)[nerv] info: create layer: softmaxL +(17:57:13 2015-12-23)[nerv] info: create layer: recurrentL1 +(17:57:13 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(17:57:13 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER24 LR0.1296]: preparing layers end. +[SCHEDULER ITER24 LR0.1296]: Generate and initing TNN ... +(17:57:13 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -6168,253 +7007,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER24 LR1]: Initing TNN end. -===ITERATION 24 LR 1.000000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER24 LR1]: 40092 words processed Wed Nov 18 12:00:20 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.068316. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47877 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96281 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.56624 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12104 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12218 clock time -[SCHEDULER ITER24 LR1]: 80099 words processed Wed Nov 18 12:00:32 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.066020. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47677 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94765 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.49793 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59499 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10381 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12061 clock time -[SCHEDULER ITER24 LR1]: 120004 words processed Wed Nov 18 12:00:44 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.061220. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47017 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92434 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44798 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58124 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09787 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11998 clock time -[SCHEDULER ITER24 LR1]: 160114 words processed Wed Nov 18 12:00:56 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.063568. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47065 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93277 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47973 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59820 clock time +(17:57:13 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:57:13 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:57:13 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:57:13 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:57:13 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:57:13 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER24 LR0.1296]: Initing TNN end. +===ITERATION 24 LR 0.129600=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(17:57:13 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(17:57:13 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(17:57:13 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(17:57:13 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(17:57:13 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(17:57:13 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(17:57:13 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER24 LR0.1296]: 40092 words processed Wed Dec 23 17:57:29 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.965882. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48181 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95396 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74164 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77716 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11071 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14840 clock time +[SCHEDULER ITER24 LR0.1296]: 80099 words processed Wed Dec 23 17:57:45 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.960740. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48107 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92838 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66935 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75026 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09704 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14714 clock time +[SCHEDULER ITER24 LR0.1296]: 120004 words processed Wed Dec 23 17:58:01 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.954928. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48032 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93080 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75234 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09887 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14736 clock time +[SCHEDULER ITER24 LR0.1296]: 160114 words processed Wed Dec 23 17:58:17 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.954825. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47999 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94979 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72206 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77127 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10508 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14785 clock time +[SCHEDULER ITER24 LR0.1296]: 200066 words processed Wed Dec 23 17:58:33 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.953977. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47487 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92822 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66332 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74978 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09627 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14721 clock time +[SCHEDULER ITER24 LR0.1296]: 240045 words processed Wed Dec 23 17:58:49 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.950017. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47734 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95298 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70986 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76023 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10127 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14882 clock time +[SCHEDULER ITER24 LR0.1296]: 280057 words processed Wed Dec 23 17:59:05 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.945615. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47740 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95046 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72830 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77395 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10620 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12011 clock time -[SCHEDULER ITER24 LR1]: 200066 words processed Wed Nov 18 12:01:08 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.062954. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46731 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92860 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45241 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58523 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09823 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12012 clock time -[SCHEDULER ITER24 LR1]: 240045 words processed Wed Nov 18 12:01:20 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.059629. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46771 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92761 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45048 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58407 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09896 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11996 clock time -[SCHEDULER ITER24 LR1]: 280057 words processed Wed Nov 18 12:01:32 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.055936. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47295 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95521 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.53857 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61370 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11431 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12263 clock time -[SCHEDULER ITER24 LR1]: 320106 words processed Wed Nov 18 12:01:44 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.054511. + [global_conf.timer]: time spent on tnn_afterprocess:0.14872 clock time +[SCHEDULER ITER24 LR0.1296]: 320106 words processed Wed Dec 23 17:59:21 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.943467. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47473 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94845 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71807 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77162 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10490 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14771 clock time +[SCHEDULER ITER24 LR0.1296]: 360024 words processed Wed Dec 23 17:59:37 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.941852. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47036 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92855 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66344 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74982 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09791 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14729 clock time +[SCHEDULER ITER24 LR0.1296]: 400089 words processed Wed Dec 23 17:59:53 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.939177. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47211 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95164 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71629 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77057 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10507 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14767 clock time +[SCHEDULER ITER24 LR0.1296]: 440067 words processed Wed Dec 23 18:00:09 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.939704. [global_conf.timer]: time spent on tnn_beforeprocess:0.47122 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95186 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.52118 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60969 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10843 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12092 clock time -[SCHEDULER ITER24 LR1]: 360024 words processed Wed Nov 18 12:01:56 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.053542. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46591 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93718 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47571 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59277 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10131 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12139 clock time -[SCHEDULER ITER24 LR1]: 400089 words processed Wed Nov 18 12:02:08 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.051705. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46908 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95995 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.53295 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61222 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11164 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12173 clock time -[SCHEDULER ITER24 LR1]: 440067 words processed Wed Nov 18 12:02:20 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.052707. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46296 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92568 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44456 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58493 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09915 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12008 clock time -[SCHEDULER ITER24 LR1]: 480051 words processed Wed Nov 18 12:02:32 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.053509. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46330 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93769 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46468 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58995 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10109 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12061 clock time -[SCHEDULER ITER24 LR1]: 520140 words processed Wed Nov 18 12:02:44 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.054860. + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93723 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68022 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75810 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09854 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14730 clock time +[SCHEDULER ITER24 LR0.1296]: 480051 words processed Wed Dec 23 18:00:25 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.940249. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47222 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93246 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66793 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75240 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09770 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14712 clock time +[SCHEDULER ITER24 LR0.1296]: 520140 words processed Wed Dec 23 18:00:41 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.939741. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47189 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94972 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71324 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76949 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10501 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14777 clock time +[SCHEDULER ITER24 LR0.1296]: 560132 words processed Wed Dec 23 18:00:57 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.939290. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47136 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92933 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65998 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09626 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14728 clock time +[SCHEDULER ITER24 LR0.1296]: 600118 words processed Wed Dec 23 18:01:13 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.937129. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47128 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92896 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66445 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75231 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09752 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14772 clock time +[SCHEDULER ITER24 LR0.1296]: 640090 words processed Wed Dec 23 18:01:29 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.935002. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47042 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92877 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65841 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74919 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09712 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14712 clock time +[SCHEDULER ITER24 LR0.1296]: 680075 words processed Wed Dec 23 18:01:45 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.933847. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46995 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92965 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65968 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75035 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09688 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14735 clock time +[SCHEDULER ITER24 LR0.1296]: 720043 words processed Wed Dec 23 18:02:01 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.932688. [global_conf.timer]: time spent on tnn_beforeprocess:0.47046 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96227 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.54502 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61623 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11317 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12316 clock time -[SCHEDULER ITER24 LR1]: 560132 words processed Wed Nov 18 12:02:56 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.054986. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47283 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96737 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.54510 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61114 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11297 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12305 clock time -[SCHEDULER ITER24 LR1]: 600118 words processed Wed Nov 18 12:03:08 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.053946. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46986 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96495 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.53254 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60815 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10816 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12281 clock time -[SCHEDULER ITER24 LR1]: 640090 words processed Wed Nov 18 12:03:20 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.052995. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46295 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92514 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43422 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58277 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09199 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12022 clock time -[SCHEDULER ITER24 LR1]: 680075 words processed Wed Nov 18 12:03:32 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.052435. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46362 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92758 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43855 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58113 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09607 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11996 clock time -[SCHEDULER ITER24 LR1]: 720043 words processed Wed Nov 18 12:03:44 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.052556. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46414 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92571 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44748 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58451 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09938 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12018 clock time -[SCHEDULER ITER24 LR1]: 760012 words processed Wed Nov 18 12:03:56 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.051749. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46523 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94188 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47211 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59200 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09732 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12069 clock time -[SCHEDULER ITER24 LR1]: 800113 words processed Wed Nov 18 12:04:08 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.052259. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46838 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96079 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.53950 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61532 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11421 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12221 clock time -[SCHEDULER ITER24 LR1]: 840089 words processed Wed Nov 18 12:04:20 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.051851. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46453 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93744 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46766 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59173 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10032 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12048 clock time -[SCHEDULER ITER24 LR1]: 880052 words processed Wed Nov 18 12:04:32 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.052134. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46421 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92912 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45550 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58541 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10261 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12032 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93238 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66340 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74960 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09577 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14744 clock time +[SCHEDULER ITER24 LR0.1296]: 760012 words processed Wed Dec 23 18:02:17 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.930803. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47031 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92831 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66164 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75082 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09696 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14739 clock time +[SCHEDULER ITER24 LR0.1296]: 800113 words processed Wed Dec 23 18:02:33 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.930484. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47262 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94994 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71601 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77121 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10486 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14812 clock time +[SCHEDULER ITER24 LR0.1296]: 840089 words processed Wed Dec 23 18:02:49 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.929270. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92837 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65671 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74904 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09591 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14697 clock time +[SCHEDULER ITER24 LR0.1296]: 880052 words processed Wed Dec 23 18:03:05 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-1.928849. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47033 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93321 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67321 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75425 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09942 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14720 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER24 LR1]: Displaying result: -[SCHEDULER ITER24 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 124.48792856202> <PPL_OOV 112.77495891323> <LOGP -1907714.3307067> -[SCHEDULER ITER24 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER24 LR1]: shuffling training file +(18:03:08 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER24 LR0.1296]: Displaying result: +[SCHEDULER ITER24 LR0.1296]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 93.484204845991> <PPL_OOV 84.883825756473> <LOGP -1793014.4519871> +[SCHEDULER ITER24 LR0.1296]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER24 LR0.1296]: shuffling training file ===PEEK ON TEST 24=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER24 LR1]: 40087 words processed Wed Nov 18 12:04:41 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.189143. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47170 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94163 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.64191 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11574 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:03:10 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:03:10 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:03:10 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:03:10 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:03:10 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:03:10 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:03:10 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER24 LR0.1296]: 40087 words processed Wed Dec 23 18:03:18 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-2.119305. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49384 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.11295 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.86840 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14345 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER24 LR1]: Displaying result: -[SCHEDULER ITER24 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 177.06047169718> <PPL_OOV 155.12796790669> <LOGP -180578.58523182> -[SCHEDULER ITER24 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(18:03:26 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER24 LR0.1296]: Displaying result: +[SCHEDULER ITER24 LR0.1296]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 150.95495538551> <PPL_OOV 131.8947936776> <LOGP -174770.36542438> +[SCHEDULER ITER24 LR0.1296]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 24=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER24 LR1]: 40095 words processed Wed Nov 18 12:04:50 2015. - [SCHEDULER ITER24 LR1]: log prob per sample :-2.256580. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47267 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94236 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.64526 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11744 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:03:26 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:03:26 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:03:26 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:03:26 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:03:26 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:03:26 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:03:26 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER24 LR0.1296]: 40095 words processed Wed Dec 23 18:03:34 2015. + [SCHEDULER ITER24 LR0.1296]: log prob per sample :-2.192698. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49301 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17164 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92510 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14296 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER24 LR1]: Displaying result: -[SCHEDULER ITER24 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 181.96453837948> <PPL_OOV 163.32705467376> <LOGP -163235.16769627> -[SCHEDULER ITER24 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER24 LR1]: PPL did not improve, rejected, copying param file of last iter... - -[SCHEDULER ITER25 LR0.6]: preparing parameters... -[SCHEDULER ITER25 LR0.6]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.24... +(18:03:40 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER24 LR0.1296]: Displaying result: +[SCHEDULER ITER24 LR0.1296]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 155.64142589889> <PPL_OOV 139.28377072933> <LOGP -158134.10203143> +[SCHEDULER ITER24 LR0.1296]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER24 LR0.1296]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.24... + +[SCHEDULER ITER25 LR0.07776]: preparing parameters... +[SCHEDULER ITER25 LR0.07776]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.24... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 1 from 1064210 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 1067867 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 35817408 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 35933682 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 5 from 70464168 -[SCHEDULER ITER25 LR0.6]: preparing parameters end. -[SCHEDULER ITER25 LR0.6]: preparing layers... -(12:04:58 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:04:58 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:04:58 2015-11-18)[nerv] info: create layer: combinerL1 -(12:04:58 2015-11-18)[nerv] info: create layer: outputL -(12:04:58 2015-11-18)[nerv] info: create layer: softmaxL -(12:04:58 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER25 LR0.6]: preparing layers end. -[SCHEDULER ITER25 LR0.6]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3665 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 119934 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1183462 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35919227 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70456356 +[SCHEDULER ITER25 LR0.07776]: preparing parameters end. +[SCHEDULER ITER25 LR0.07776]: preparing layers... +(18:03:46 2015-12-23)[nerv] info: create layer: sigmoidL1 +(18:03:46 2015-12-23)[nerv] info: create layer: combinerL1 +(18:03:46 2015-12-23)[nerv] info: create layer: selectL1 +(18:03:46 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(18:03:46 2015-12-23)[nerv] info: create layer: outputL +(18:03:46 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(18:03:46 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(18:03:46 2015-12-23)[nerv] info: create layer: softmaxL +(18:03:46 2015-12-23)[nerv] info: create layer: recurrentL1 +(18:03:46 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(18:03:46 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER25 LR0.07776]: preparing layers end. +[SCHEDULER ITER25 LR0.07776]: Generate and initing TNN ... +(18:03:46 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -6430,253 +7305,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER25 LR0.6]: Initing TNN end. -===ITERATION 25 LR 0.600000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER25 LR0.6]: 40092 words processed Wed Nov 18 12:05:09 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.031786. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47559 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92421 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.49377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61521 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10529 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12154 clock time -[SCHEDULER ITER25 LR0.6]: 80099 words processed Wed Nov 18 12:05:21 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.027880. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47539 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91322 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44897 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59323 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09745 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12134 clock time -[SCHEDULER ITER25 LR0.6]: 120004 words processed Wed Nov 18 12:05:33 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.021698. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47358 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91289 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59259 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09935 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12134 clock time -[SCHEDULER ITER25 LR0.6]: 160114 words processed Wed Nov 18 12:05:45 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.022082. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47522 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93109 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.50085 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61470 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10597 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12159 clock time -[SCHEDULER ITER25 LR0.6]: 200066 words processed Wed Nov 18 12:05:57 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.021204. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46811 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89821 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58945 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09367 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12050 clock time -[SCHEDULER ITER25 LR0.6]: 240045 words processed Wed Nov 18 12:06:09 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.017220. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46718 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89326 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40761 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58620 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09416 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11973 clock time -[SCHEDULER ITER25 LR0.6]: 280057 words processed Wed Nov 18 12:06:21 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.013365. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47118 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91875 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46655 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60666 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09618 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12136 clock time -[SCHEDULER ITER25 LR0.6]: 320106 words processed Wed Nov 18 12:06:33 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.011554. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46882 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91896 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47497 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60626 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10743 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12162 clock time -[SCHEDULER ITER25 LR0.6]: 360024 words processed Wed Nov 18 12:06:45 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.009923. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46184 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89187 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40414 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58473 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09742 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11935 clock time -[SCHEDULER ITER25 LR0.6]: 400089 words processed Wed Nov 18 12:06:57 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.007376. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89847 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43583 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60084 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10563 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11964 clock time -[SCHEDULER ITER25 LR0.6]: 440067 words processed Wed Nov 18 12:07:09 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.008010. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46460 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90775 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42989 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58809 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10138 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12080 clock time -[SCHEDULER ITER25 LR0.6]: 480051 words processed Wed Nov 18 12:07:21 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.008517. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46515 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90557 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42563 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58875 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09904 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12044 clock time -[SCHEDULER ITER25 LR0.6]: 520140 words processed Wed Nov 18 12:07:33 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.008299. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46661 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91944 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46670 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60524 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10376 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12110 clock time -[SCHEDULER ITER25 LR0.6]: 560132 words processed Wed Nov 18 12:07:45 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.007875. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46230 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89746 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41263 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58678 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10029 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12003 clock time -[SCHEDULER ITER25 LR0.6]: 600118 words processed Wed Nov 18 12:07:57 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.005749. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46345 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89997 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59533 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10248 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12171 clock time -[SCHEDULER ITER25 LR0.6]: 640090 words processed Wed Nov 18 12:08:09 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.004148. - [global_conf.timer]: time spent on tnn_beforeprocess:0.49615 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.08220 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.86480 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.71813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14119 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13261 clock time -[SCHEDULER ITER25 LR0.6]: 680075 words processed Wed Nov 18 12:08:21 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.003208. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47627 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95697 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.56682 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.63987 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10643 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12439 clock time -[SCHEDULER ITER25 LR0.6]: 720043 words processed Wed Nov 18 12:08:33 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.002754. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48075 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99602 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.68664 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67512 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12576 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12818 clock time -[SCHEDULER ITER25 LR0.6]: 760012 words processed Wed Nov 18 12:08:45 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.001453. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46981 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94694 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.55503 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.63410 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11301 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12533 clock time -[SCHEDULER ITER25 LR0.6]: 800113 words processed Wed Nov 18 12:08:57 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.001667. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48087 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99964 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.68295 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66725 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13223 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12834 clock time -[SCHEDULER ITER25 LR0.6]: 840089 words processed Wed Nov 18 12:09:09 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.000776. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92299 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48215 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60408 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10920 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12288 clock time -[SCHEDULER ITER25 LR0.6]: 880052 words processed Wed Nov 18 12:09:21 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.000744. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46297 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89681 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42192 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59114 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10145 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11999 clock time +(18:03:46 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:03:46 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:03:46 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:03:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:03:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:03:46 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER25 LR0.07776]: Initing TNN end. +===ITERATION 25 LR 0.077760=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:03:46 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:03:46 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:03:46 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:03:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:03:46 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:03:46 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:03:46 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER25 LR0.07776]: 40092 words processed Wed Dec 23 18:04:02 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.954490. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48163 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95711 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74594 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77281 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11452 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14839 clock time +[SCHEDULER ITER25 LR0.07776]: 80099 words processed Wed Dec 23 18:04:18 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.949383. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48028 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93089 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74522 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10069 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14702 clock time +[SCHEDULER ITER25 LR0.07776]: 120004 words processed Wed Dec 23 18:04:34 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.943709. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47859 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92876 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74364 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09873 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14712 clock time +[SCHEDULER ITER25 LR0.07776]: 160114 words processed Wed Dec 23 18:04:50 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.943688. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47894 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95078 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71737 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76522 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10547 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14745 clock time +[SCHEDULER ITER25 LR0.07776]: 200066 words processed Wed Dec 23 18:05:06 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.942923. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47595 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93731 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67750 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74858 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09882 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14745 clock time +[SCHEDULER ITER25 LR0.07776]: 240045 words processed Wed Dec 23 18:05:22 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.939047. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47479 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93045 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66074 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74233 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09854 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14676 clock time +[SCHEDULER ITER25 LR0.07776]: 280057 words processed Wed Dec 23 18:05:38 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.934676. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47675 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95125 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71852 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76450 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10628 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14740 clock time +[SCHEDULER ITER25 LR0.07776]: 320106 words processed Wed Dec 23 18:05:54 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.932560. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47344 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95129 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71553 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76570 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10522 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14771 clock time +[SCHEDULER ITER25 LR0.07776]: 360024 words processed Wed Dec 23 18:06:10 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.930959. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47133 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93163 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66832 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74737 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09880 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14776 clock time +[SCHEDULER ITER25 LR0.07776]: 400089 words processed Wed Dec 23 18:06:26 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.928260. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47179 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95179 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71039 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76360 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10552 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14742 clock time +[SCHEDULER ITER25 LR0.07776]: 440067 words processed Wed Dec 23 18:06:42 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.928769. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47143 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93049 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66258 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74693 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09868 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14715 clock time +[SCHEDULER ITER25 LR0.07776]: 480051 words processed Wed Dec 23 18:06:58 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.929290. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47048 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93099 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65833 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74665 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09682 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14683 clock time +[SCHEDULER ITER25 LR0.07776]: 520140 words processed Wed Dec 23 18:07:14 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.928757. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47272 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95297 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71982 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76853 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10618 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14767 clock time +[SCHEDULER ITER25 LR0.07776]: 560132 words processed Wed Dec 23 18:07:30 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.928305. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47245 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95790 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70663 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75786 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10131 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14764 clock time +[SCHEDULER ITER25 LR0.07776]: 600118 words processed Wed Dec 23 18:07:46 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.926125. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92797 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65470 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74329 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09874 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14675 clock time +[SCHEDULER ITER25 LR0.07776]: 640090 words processed Wed Dec 23 18:08:02 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.923980. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47473 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97453 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73655 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76282 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10310 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14948 clock time +[SCHEDULER ITER25 LR0.07776]: 680075 words processed Wed Dec 23 18:08:18 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.922791. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47072 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93177 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65874 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74466 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09754 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14682 clock time +[SCHEDULER ITER25 LR0.07776]: 720043 words processed Wed Dec 23 18:08:34 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.921594. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47017 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92972 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65639 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74361 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09755 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14667 clock time +[SCHEDULER ITER25 LR0.07776]: 760012 words processed Wed Dec 23 18:08:50 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.919637. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46992 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93003 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65784 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74389 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09829 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14683 clock time +[SCHEDULER ITER25 LR0.07776]: 800113 words processed Wed Dec 23 18:09:06 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.919291. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47310 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95806 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72175 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76782 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10393 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14784 clock time +[SCHEDULER ITER25 LR0.07776]: 840089 words processed Wed Dec 23 18:09:22 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.918034. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46935 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93141 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65608 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74370 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09784 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14689 clock time +[SCHEDULER ITER25 LR0.07776]: 880052 words processed Wed Dec 23 18:09:38 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-1.917560. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46984 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93118 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65952 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74473 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09731 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14725 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER25 LR0.6]: Displaying result: -[SCHEDULER ITER25 LR0.6]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 110.67448591716> <PPL_OOV 100.1769714556> <LOGP -1859891.8295212> -[SCHEDULER ITER25 LR0.6]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER25 LR0.6]: shuffling training file +(18:09:41 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER25 LR0.07776]: Displaying result: +[SCHEDULER ITER25 LR0.07776]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 91.023870840216> <PPL_OOV 82.703545713429> <LOGP -1782509.3378055> +[SCHEDULER ITER25 LR0.07776]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER25 LR0.07776]: shuffling training file ===PEEK ON TEST 25=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER25 LR0.6]: 40087 words processed Wed Nov 18 12:09:30 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.159875. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47367 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93342 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.63546 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11656 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:09:43 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:09:43 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:09:43 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:09:43 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:09:43 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:09:43 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:09:43 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER25 LR0.07776]: 40087 words processed Wed Dec 23 18:09:51 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-2.113922. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49612 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.15157 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.91326 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14516 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER25 LR0.6]: Displaying result: -[SCHEDULER ITER25 LR0.6]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 165.44666654399> <PPL_OOV 145.17179861502> <LOGP -178203.95449233> -[SCHEDULER ITER25 LR0.6]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(18:09:59 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER25 LR0.07776]: Displaying result: +[SCHEDULER ITER25 LR0.07776]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 149.22506055553> <PPL_OOV 130.38662370638> <LOGP -174358.65945783> +[SCHEDULER ITER25 LR0.07776]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 25=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER25 LR0.6]: 40095 words processed Wed Nov 18 12:09:39 2015. - [SCHEDULER ITER25 LR0.6]: log prob per sample :-2.230972. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47256 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93270 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.63300 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11649 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:09:59 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:09:59 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:09:59 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:09:59 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:09:59 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:09:59 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:09:59 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER25 LR0.07776]: 40095 words processed Wed Dec 23 18:10:07 2015. + [SCHEDULER ITER25 LR0.07776]: log prob per sample :-2.187959. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49297 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.18857 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.94415 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14257 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER25 LR0.6]: Displaying result: -[SCHEDULER ITER25 LR0.6]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 170.02460806547> <PPL_OOV 152.80634249262> <LOGP -161102.26824489> -[SCHEDULER ITER25 LR0.6]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER25 LR0.6]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.25... - -[SCHEDULER ITER26 LR0.36]: preparing parameters... -[SCHEDULER ITER26 LR0.36]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.25... +(18:10:13 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER25 LR0.07776]: Displaying result: +[SCHEDULER ITER25 LR0.07776]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 153.83591919324> <PPL_OOV 137.6811986416> <LOGP -157763.39331874> +[SCHEDULER ITER25 LR0.07776]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER25 LR0.07776]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.25... + +[SCHEDULER ITER26 LR0.046656]: preparing parameters... +[SCHEDULER ITER26 LR0.046656]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.25... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 1 from 34528953 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 34532610 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 69287801 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 69404050 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 5 from 70467695 -[SCHEDULER ITER26 LR0.36]: preparing parameters end. -[SCHEDULER ITER26 LR0.36]: preparing layers... -(12:09:48 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:09:48 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:09:48 2015-11-18)[nerv] info: create layer: combinerL1 -(12:09:48 2015-11-18)[nerv] info: create layer: outputL -(12:09:48 2015-11-18)[nerv] info: create layer: softmaxL -(12:09:48 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER26 LR0.36]: preparing layers end. -[SCHEDULER ITER26 LR0.36]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3663 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 119929 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1183260 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35918608 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70455672 +[SCHEDULER ITER26 LR0.046656]: preparing parameters end. +[SCHEDULER ITER26 LR0.046656]: preparing layers... +(18:10:19 2015-12-23)[nerv] info: create layer: sigmoidL1 +(18:10:19 2015-12-23)[nerv] info: create layer: combinerL1 +(18:10:19 2015-12-23)[nerv] info: create layer: selectL1 +(18:10:19 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(18:10:19 2015-12-23)[nerv] info: create layer: outputL +(18:10:19 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(18:10:19 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(18:10:19 2015-12-23)[nerv] info: create layer: softmaxL +(18:10:19 2015-12-23)[nerv] info: create layer: recurrentL1 +(18:10:19 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(18:10:19 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER26 LR0.046656]: preparing layers end. +[SCHEDULER ITER26 LR0.046656]: Generate and initing TNN ... +(18:10:19 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -6692,253 +7603,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER26 LR0.36]: Initing TNN end. -===ITERATION 26 LR 0.360000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER26 LR0.36]: 40092 words processed Wed Nov 18 12:10:00 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.996688. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47915 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95989 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.56956 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62875 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12190 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12261 clock time -[SCHEDULER ITER26 LR0.36]: 80099 words processed Wed Nov 18 12:10:12 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.993541. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47391 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91974 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44910 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58699 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10195 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11975 clock time -[SCHEDULER ITER26 LR0.36]: 120004 words processed Wed Nov 18 12:10:24 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.987958. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91489 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43678 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58234 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09952 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11917 clock time -[SCHEDULER ITER26 LR0.36]: 160114 words processed Wed Nov 18 12:10:36 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.988377. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47236 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92116 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46188 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59698 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10566 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11931 clock time -[SCHEDULER ITER26 LR0.36]: 200066 words processed Wed Nov 18 12:10:48 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.987455. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46790 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91231 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42714 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58112 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10069 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11876 clock time -[SCHEDULER ITER26 LR0.36]: 240045 words processed Wed Nov 18 12:11:00 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.983531. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46960 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92541 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46234 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59207 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10635 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11997 clock time -[SCHEDULER ITER26 LR0.36]: 280057 words processed Wed Nov 18 12:11:12 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.979726. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47193 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94011 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51075 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61099 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11318 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12103 clock time -[SCHEDULER ITER26 LR0.36]: 320106 words processed Wed Nov 18 12:11:24 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.977856. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47081 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94043 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.50615 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60942 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11170 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12083 clock time -[SCHEDULER ITER26 LR0.36]: 360024 words processed Wed Nov 18 12:11:36 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.976151. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46723 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93346 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46853 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59249 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10353 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11945 clock time -[SCHEDULER ITER26 LR0.36]: 400089 words processed Wed Nov 18 12:11:48 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.973558. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46809 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93962 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.50217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60705 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11601 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12037 clock time -[SCHEDULER ITER26 LR0.36]: 440067 words processed Wed Nov 18 12:12:00 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.974075. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46445 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92254 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44463 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58898 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10076 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11973 clock time -[SCHEDULER ITER26 LR0.36]: 480051 words processed Wed Nov 18 12:12:12 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.974452. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46744 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92710 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46191 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59450 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10487 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12041 clock time -[SCHEDULER ITER26 LR0.36]: 520140 words processed Wed Nov 18 12:12:24 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.974107. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46733 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93118 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48843 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60493 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11402 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12034 clock time -[SCHEDULER ITER26 LR0.36]: 560132 words processed Wed Nov 18 12:12:36 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.973552. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46368 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91026 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42476 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58186 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10460 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11945 clock time -[SCHEDULER ITER26 LR0.36]: 600118 words processed Wed Nov 18 12:12:48 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.971346. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46390 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91147 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42577 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58289 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10310 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11931 clock time -[SCHEDULER ITER26 LR0.36]: 640090 words processed Wed Nov 18 12:13:00 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.969490. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91097 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41968 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58116 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09955 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11912 clock time -[SCHEDULER ITER26 LR0.36]: 680075 words processed Wed Nov 18 12:13:12 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.968381. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46427 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91362 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43152 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58302 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10576 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11939 clock time -[SCHEDULER ITER26 LR0.36]: 720043 words processed Wed Nov 18 12:13:24 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.967690. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46487 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91613 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43145 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58442 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09900 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11942 clock time -[SCHEDULER ITER26 LR0.36]: 760012 words processed Wed Nov 18 12:13:36 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.966082. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46423 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91284 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41629 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58204 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09114 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11908 clock time -[SCHEDULER ITER26 LR0.36]: 800113 words processed Wed Nov 18 12:13:48 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.966133. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46562 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92353 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46595 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59862 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10959 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11979 clock time -[SCHEDULER ITER26 LR0.36]: 840089 words processed Wed Nov 18 12:14:00 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.965067. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46287 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91166 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42094 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58290 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09866 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11944 clock time -[SCHEDULER ITER26 LR0.36]: 880052 words processed Wed Nov 18 12:14:12 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-1.964871. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46517 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91571 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43677 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58790 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09944 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11961 clock time +(18:10:19 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:10:19 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:10:19 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:10:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:10:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:10:19 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER26 LR0.046656]: Initing TNN end. +===ITERATION 26 LR 0.046656=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:10:19 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:10:19 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:10:19 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:10:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:10:19 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:10:19 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:10:19 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER26 LR0.046656]: 40092 words processed Wed Dec 23 18:10:35 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.946425. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48148 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96435 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74466 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76818 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11180 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14730 clock time +[SCHEDULER ITER26 LR0.046656]: 80099 words processed Wed Dec 23 18:10:51 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.941430. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48140 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93148 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66989 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74448 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09882 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14679 clock time +[SCHEDULER ITER26 LR0.046656]: 120004 words processed Wed Dec 23 18:11:07 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.935944. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47961 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93836 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68606 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74889 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09898 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14737 clock time +[SCHEDULER ITER26 LR0.046656]: 160114 words processed Wed Dec 23 18:11:23 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.936126. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47965 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95352 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71810 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76319 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10503 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14741 clock time +[SCHEDULER ITER26 LR0.046656]: 200066 words processed Wed Dec 23 18:11:39 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.935456. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47466 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93050 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65993 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74199 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09767 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14641 clock time +[SCHEDULER ITER26 LR0.046656]: 240045 words processed Wed Dec 23 18:11:55 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.931644. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47693 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94233 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68437 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74960 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09831 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14734 clock time +[SCHEDULER ITER26 LR0.046656]: 280057 words processed Wed Dec 23 18:12:11 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.927324. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47802 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96817 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74894 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77228 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10858 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time +[SCHEDULER ITER26 LR0.046656]: 320106 words processed Wed Dec 23 18:12:27 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.925248. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47463 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95221 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71419 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76434 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10400 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14693 clock time +[SCHEDULER ITER26 LR0.046656]: 360024 words processed Wed Dec 23 18:12:43 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.923663. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47068 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93151 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66111 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74435 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09679 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14673 clock time +[SCHEDULER ITER26 LR0.046656]: 400089 words processed Wed Dec 23 18:12:59 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.920972. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47265 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95223 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71117 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76510 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10313 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14725 clock time +[SCHEDULER ITER26 LR0.046656]: 440067 words processed Wed Dec 23 18:13:15 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.921488. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47067 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93523 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66364 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74672 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09560 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14699 clock time +[SCHEDULER ITER26 LR0.046656]: 480051 words processed Wed Dec 23 18:13:31 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.922015. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47144 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94417 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.68189 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75244 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09818 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14713 clock time +[SCHEDULER ITER26 LR0.046656]: 520140 words processed Wed Dec 23 18:13:47 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.921486. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47341 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95991 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72683 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10402 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14745 clock time +[SCHEDULER ITER26 LR0.046656]: 560132 words processed Wed Dec 23 18:14:03 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.921060. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47134 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94187 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67497 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74854 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09790 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14755 clock time +[SCHEDULER ITER26 LR0.046656]: 600118 words processed Wed Dec 23 18:14:19 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.918881. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93134 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65672 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74494 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09605 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14681 clock time +[SCHEDULER ITER26 LR0.046656]: 640090 words processed Wed Dec 23 18:14:35 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.916733. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47004 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93270 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65789 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74457 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09634 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14630 clock time +[SCHEDULER ITER26 LR0.046656]: 680075 words processed Wed Dec 23 18:14:51 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.915531. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46983 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93184 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65788 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74504 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09721 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14663 clock time +[SCHEDULER ITER26 LR0.046656]: 720043 words processed Wed Dec 23 18:15:07 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.914303. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47024 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93977 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67453 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75007 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09804 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14667 clock time +[SCHEDULER ITER26 LR0.046656]: 760012 words processed Wed Dec 23 18:15:23 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.912310. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47000 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93639 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66732 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74660 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09782 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14716 clock time +[SCHEDULER ITER26 LR0.046656]: 800113 words processed Wed Dec 23 18:15:39 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.911964. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47213 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95363 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70994 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10302 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14696 clock time +[SCHEDULER ITER26 LR0.046656]: 840089 words processed Wed Dec 23 18:15:55 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.910685. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47322 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95984 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70907 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75718 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10060 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14835 clock time +[SCHEDULER ITER26 LR0.046656]: 880052 words processed Wed Dec 23 18:16:11 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-1.910186. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47446 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97775 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.74131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76392 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10240 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14888 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER26 LR0.36]: Displaying result: -[SCHEDULER ITER26 LR0.36]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 101.74751036446> <PPL_OOV 92.223276870422> <LOGP -1826494.1615467> -[SCHEDULER ITER26 LR0.36]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER26 LR0.36]: shuffling training file +(18:16:14 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER26 LR0.046656]: Displaying result: +[SCHEDULER ITER26 LR0.046656]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 89.453136908795> <PPL_OOV 81.3100681081> <LOGP -1775649.152788> +[SCHEDULER ITER26 LR0.046656]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER26 LR0.046656]: shuffling training file ===PEEK ON TEST 26=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER26 LR0.36]: 40087 words processed Wed Nov 18 12:14:21 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-2.142643. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92402 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62097 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11535 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:16:16 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:16:16 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:16:16 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:16:16 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:16:16 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:16:16 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:16:16 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER26 LR0.046656]: 40087 words processed Wed Dec 23 18:16:24 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-2.110239. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49699 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17456 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.93878 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14577 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER26 LR0.36]: Displaying result: -[SCHEDULER ITER26 LR0.36]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 159.3229023722> <PPL_OOV 139.5092618451> <LOGP -176779.62865589> -[SCHEDULER ITER26 LR0.36]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(18:16:32 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER26 LR0.046656]: Displaying result: +[SCHEDULER ITER26 LR0.046656]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 147.89539006091> <PPL_OOV 129.29384628988> <LOGP -174057.36260651> +[SCHEDULER ITER26 LR0.046656]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 26=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER26 LR0.36]: 40095 words processed Wed Nov 18 12:14:30 2015. - [SCHEDULER ITER26 LR0.36]: log prob per sample :-2.215991. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47448 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92963 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62958 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11549 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:16:32 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:16:32 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:16:32 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:16:32 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:16:32 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:16:32 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:16:32 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER26 LR0.046656]: 40095 words processed Wed Dec 23 18:16:40 2015. + [SCHEDULER ITER26 LR0.046656]: log prob per sample :-2.184406. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49154 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17315 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92534 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14220 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER26 LR0.36]: Displaying result: -[SCHEDULER ITER26 LR0.36]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 163.83398316372> <PPL_OOV 146.95799744127> <LOGP -159852.17229954> -[SCHEDULER ITER26 LR0.36]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER26 LR0.36]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.26... - -[SCHEDULER ITER27 LR0.216]: preparing parameters... -[SCHEDULER ITER27 LR0.216]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.26... +(18:16:46 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER26 LR0.046656]: Displaying result: +[SCHEDULER ITER26 LR0.046656]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 152.50878688257> <PPL_OOV 136.55774330169> <LOGP -157500.93258859> +[SCHEDULER ITER26 LR0.046656]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER26 LR0.046656]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.26... + +[SCHEDULER ITER27 LR0.0279936]: preparing parameters... +[SCHEDULER ITER27 LR0.0279936]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.26... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 1 from 1063239 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 1066894 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 35823725 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 35939977 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 5 from 70467798 -[SCHEDULER ITER27 LR0.216]: preparing parameters end. -[SCHEDULER ITER27 LR0.216]: preparing layers... -(12:14:39 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:14:39 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:14:39 2015-11-18)[nerv] info: create layer: combinerL1 -(12:14:39 2015-11-18)[nerv] info: create layer: outputL -(12:14:39 2015-11-18)[nerv] info: create layer: softmaxL -(12:14:39 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER27 LR0.216]: preparing layers end. -[SCHEDULER ITER27 LR0.216]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3663 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 119932 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1183153 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35918159 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70455146 +[SCHEDULER ITER27 LR0.0279936]: preparing parameters end. +[SCHEDULER ITER27 LR0.0279936]: preparing layers... +(18:16:52 2015-12-23)[nerv] info: create layer: sigmoidL1 +(18:16:52 2015-12-23)[nerv] info: create layer: combinerL1 +(18:16:52 2015-12-23)[nerv] info: create layer: selectL1 +(18:16:52 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(18:16:52 2015-12-23)[nerv] info: create layer: outputL +(18:16:52 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(18:16:52 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(18:16:52 2015-12-23)[nerv] info: create layer: softmaxL +(18:16:52 2015-12-23)[nerv] info: create layer: recurrentL1 +(18:16:52 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(18:16:52 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER27 LR0.0279936]: preparing layers end. +[SCHEDULER ITER27 LR0.0279936]: Generate and initing TNN ... +(18:16:52 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -6954,253 +7901,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER27 LR0.216]: Initing TNN end. -===ITERATION 27 LR 0.216000=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER27 LR0.216]: 40092 words processed Wed Nov 18 12:14:50 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.973334. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47323 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91484 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47757 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61267 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10593 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12087 clock time -[SCHEDULER ITER27 LR0.216]: 80099 words processed Wed Nov 18 12:15:02 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.970486. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47247 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89811 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42323 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59018 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09599 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12019 clock time -[SCHEDULER ITER27 LR0.216]: 120004 words processed Wed Nov 18 12:15:14 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.965229. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47104 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90382 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42997 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59004 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09348 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12053 clock time -[SCHEDULER ITER27 LR0.216]: 160114 words processed Wed Nov 18 12:15:26 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.965870. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47145 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91094 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45531 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60349 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10090 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12021 clock time -[SCHEDULER ITER27 LR0.216]: 200066 words processed Wed Nov 18 12:15:38 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.964948. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46847 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90485 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42501 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09463 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12013 clock time -[SCHEDULER ITER27 LR0.216]: 240045 words processed Wed Nov 18 12:15:50 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.961024. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46741 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90016 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41703 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58858 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09392 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11989 clock time -[SCHEDULER ITER27 LR0.216]: 280057 words processed Wed Nov 18 12:16:02 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.957283. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46741 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90229 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44591 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60225 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10394 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11995 clock time -[SCHEDULER ITER27 LR0.216]: 320106 words processed Wed Nov 18 12:16:14 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.955454. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46583 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90499 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44292 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60193 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09970 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12001 clock time -[SCHEDULER ITER27 LR0.216]: 360024 words processed Wed Nov 18 12:16:26 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.953706. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46236 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89393 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40390 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09437 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11915 clock time -[SCHEDULER ITER27 LR0.216]: 400089 words processed Wed Nov 18 12:16:38 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.951011. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46415 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90296 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44056 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60284 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10222 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11959 clock time -[SCHEDULER ITER27 LR0.216]: 440067 words processed Wed Nov 18 12:16:50 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.951515. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47001 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92909 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60908 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10588 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12211 clock time -[SCHEDULER ITER27 LR0.216]: 480051 words processed Wed Nov 18 12:17:02 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.951814. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46407 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90699 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42907 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59041 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10094 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12051 clock time -[SCHEDULER ITER27 LR0.216]: 520140 words processed Wed Nov 18 12:17:14 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.951394. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46604 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91905 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47077 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60997 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10360 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12121 clock time -[SCHEDULER ITER27 LR0.216]: 560132 words processed Wed Nov 18 12:17:26 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.950774. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46405 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90826 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42279 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59112 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09222 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12059 clock time -[SCHEDULER ITER27 LR0.216]: 600118 words processed Wed Nov 18 12:17:38 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.948562. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46344 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90077 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58681 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09426 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12025 clock time -[SCHEDULER ITER27 LR0.216]: 640090 words processed Wed Nov 18 12:17:50 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.946590. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46366 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90235 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41475 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58685 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09588 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11993 clock time -[SCHEDULER ITER27 LR0.216]: 680075 words processed Wed Nov 18 12:18:02 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.945381. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46313 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90135 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41396 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58735 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09607 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12009 clock time -[SCHEDULER ITER27 LR0.216]: 720043 words processed Wed Nov 18 12:18:14 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.944535. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46460 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91359 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44771 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59546 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10296 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12076 clock time -[SCHEDULER ITER27 LR0.216]: 760012 words processed Wed Nov 18 12:18:26 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.942761. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46678 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91923 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45344 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59768 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09793 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12117 clock time -[SCHEDULER ITER27 LR0.216]: 800113 words processed Wed Nov 18 12:18:38 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.942714. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46687 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92992 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48893 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61481 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10322 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12156 clock time -[SCHEDULER ITER27 LR0.216]: 840089 words processed Wed Nov 18 12:18:50 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.941552. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46516 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90922 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59295 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10211 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12069 clock time -[SCHEDULER ITER27 LR0.216]: 880052 words processed Wed Nov 18 12:19:02 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-1.941233. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46569 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90935 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44414 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59548 clock time +(18:16:52 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:16:52 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:16:52 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:16:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:16:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:16:52 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER27 LR0.0279936]: Initing TNN end. +===ITERATION 27 LR 0.027994=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:16:52 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:16:52 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:16:52 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:16:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:16:52 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:16:52 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:16:52 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER27 LR0.0279936]: 40092 words processed Wed Dec 23 18:17:08 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.941412. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48721 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99320 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.81376 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78858 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11868 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14970 clock time +[SCHEDULER ITER27 LR0.0279936]: 80099 words processed Wed Dec 23 18:17:24 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.936384. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92723 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66731 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74377 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10170 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14657 clock time +[SCHEDULER ITER27 LR0.0279936]: 120004 words processed Wed Dec 23 18:17:40 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.931003. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47829 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92652 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67080 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74458 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10238 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14696 clock time +[SCHEDULER ITER27 LR0.0279936]: 160114 words processed Wed Dec 23 18:17:56 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.931265. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94846 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72086 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76622 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10966 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14754 clock time +[SCHEDULER ITER27 LR0.0279936]: 200066 words processed Wed Dec 23 18:18:12 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.930654. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47436 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92924 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66883 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74703 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10178 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14678 clock time +[SCHEDULER ITER27 LR0.0279936]: 240045 words processed Wed Dec 23 18:18:28 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.926862. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47459 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92752 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66261 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74481 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10085 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14637 clock time +[SCHEDULER ITER27 LR0.0279936]: 280057 words processed Wed Dec 23 18:18:44 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.922576. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47667 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94769 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76480 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10889 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14728 clock time +[SCHEDULER ITER27 LR0.0279936]: 320106 words processed Wed Dec 23 18:19:00 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.920508. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47404 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95065 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72688 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76873 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11251 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14770 clock time +[SCHEDULER ITER27 LR0.0279936]: 360024 words processed Wed Dec 23 18:19:16 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.918927. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47106 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93079 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67136 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74708 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10269 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14717 clock time +[SCHEDULER ITER27 LR0.0279936]: 400089 words processed Wed Dec 23 18:19:32 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.916253. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47222 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94901 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71198 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10760 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14753 clock time +[SCHEDULER ITER27 LR0.0279936]: 440067 words processed Wed Dec 23 18:19:48 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.916752. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46991 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92793 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65694 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74401 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09976 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14660 clock time +[SCHEDULER ITER27 LR0.0279936]: 480051 words processed Wed Dec 23 18:20:04 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.917256. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47108 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92907 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66276 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74635 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10159 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14681 clock time +[SCHEDULER ITER27 LR0.0279936]: 520140 words processed Wed Dec 23 18:20:20 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.916713. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97782 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76463 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77833 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11211 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14774 clock time +[SCHEDULER ITER27 LR0.0279936]: 560132 words processed Wed Dec 23 18:20:36 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.916295. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47050 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92825 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74483 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10314 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12035 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14656 clock time +[SCHEDULER ITER27 LR0.0279936]: 600118 words processed Wed Dec 23 18:20:52 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.914101. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47090 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92817 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65766 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74352 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09969 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14689 clock time +[SCHEDULER ITER27 LR0.0279936]: 640090 words processed Wed Dec 23 18:21:08 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.911930. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47007 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92733 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65605 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74416 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09938 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14683 clock time +[SCHEDULER ITER27 LR0.0279936]: 680075 words processed Wed Dec 23 18:21:24 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.910703. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46961 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92784 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65492 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10018 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14670 clock time +[SCHEDULER ITER27 LR0.0279936]: 720043 words processed Wed Dec 23 18:21:40 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.909447. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46960 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92901 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65837 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74421 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09974 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14654 clock time +[SCHEDULER ITER27 LR0.0279936]: 760012 words processed Wed Dec 23 18:21:56 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.907415. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47006 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92874 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65976 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74395 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10106 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14663 clock time +[SCHEDULER ITER27 LR0.0279936]: 800113 words processed Wed Dec 23 18:22:12 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.907042. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47311 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95681 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11007 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14795 clock time +[SCHEDULER ITER27 LR0.0279936]: 840089 words processed Wed Dec 23 18:22:28 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.905726. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92860 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65852 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74564 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09964 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14675 clock time +[SCHEDULER ITER27 LR0.0279936]: 880052 words processed Wed Dec 23 18:22:44 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-1.905188. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47035 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92885 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66137 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74512 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10059 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14677 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER27 LR0.216]: Displaying result: -[SCHEDULER ITER27 LR0.216]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 96.230665927949> <PPL_OOV 87.330596039854> <LOGP -1804486.9551715> -[SCHEDULER ITER27 LR0.216]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER27 LR0.216]: shuffling training file +(18:22:47 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER27 LR0.0279936]: Displaying result: +[SCHEDULER ITER27 LR0.0279936]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 88.405824376249> <PPL_OOV 80.378855892909> <LOGP -1770998.8794815> +[SCHEDULER ITER27 LR0.0279936]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER27 LR0.0279936]: shuffling training file ===PEEK ON TEST 27=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER27 LR0.216]: 40087 words processed Wed Nov 18 12:19:11 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-2.129422. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92878 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62932 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11584 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:22:49 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:22:49 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:22:49 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:22:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:22:49 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:22:49 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:22:49 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER27 LR0.0279936]: 40087 words processed Wed Dec 23 18:22:57 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-2.108436. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49207 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.13031 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.88535 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14438 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER27 LR0.216]: Displaying result: -[SCHEDULER ITER27 LR0.216]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 155.20859848991> <PPL_OOV 135.39765175113> <LOGP -175708.70563493> -[SCHEDULER ITER27 LR0.216]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(18:23:05 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER27 LR0.0279936]: Displaying result: +[SCHEDULER ITER27 LR0.0279936]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 147.25458293433> <PPL_OOV 128.73201648926> <LOGP -173901.46414635> +[SCHEDULER ITER27 LR0.0279936]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 27=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER27 LR0.216]: 40095 words processed Wed Nov 18 12:19:20 2015. - [SCHEDULER ITER27 LR0.216]: log prob per sample :-2.204569. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47198 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92267 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62166 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11625 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:23:05 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:23:05 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:23:05 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:23:05 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:23:05 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:23:05 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:23:05 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER27 LR0.0279936]: 40095 words processed Wed Dec 23 18:23:13 2015. + [SCHEDULER ITER27 LR0.0279936]: log prob per sample :-2.182545. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49118 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.16794 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.91915 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14233 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER27 LR0.216]: Displaying result: -[SCHEDULER ITER27 LR0.216]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 159.64602319117> <PPL_OOV 142.74466017102> <LOGP -158920.33608658> -[SCHEDULER ITER27 LR0.216]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER27 LR0.216]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.27... - -[SCHEDULER ITER28 LR0.1296]: preparing parameters... -[SCHEDULER ITER28 LR0.1296]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.27... +(18:23:19 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER27 LR0.0279936]: Displaying result: +[SCHEDULER ITER27 LR0.0279936]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 151.8412756902> <PPL_OOV 135.96623873338> <LOGP -157361.87673541> +[SCHEDULER ITER27 LR0.0279936]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER27 LR0.0279936]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.27... + +[SCHEDULER ITER28 LR0.01679616]: preparing parameters... +[SCHEDULER ITER28 LR0.01679616]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.27... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 1 from 34526862 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 34530515 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 69287866 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 69404115 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 5 from 70467110 -[SCHEDULER ITER28 LR0.1296]: preparing parameters end. -[SCHEDULER ITER28 LR0.1296]: preparing layers... -(12:19:29 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:19:29 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:19:29 2015-11-18)[nerv] info: create layer: combinerL1 -(12:19:29 2015-11-18)[nerv] info: create layer: outputL -(12:19:29 2015-11-18)[nerv] info: create layer: softmaxL -(12:19:29 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER28 LR0.1296]: preparing layers end. -[SCHEDULER ITER28 LR0.1296]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3663 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 119931 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1183102 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35917947 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70454886 +[SCHEDULER ITER28 LR0.01679616]: preparing parameters end. +[SCHEDULER ITER28 LR0.01679616]: preparing layers... +(18:23:25 2015-12-23)[nerv] info: create layer: sigmoidL1 +(18:23:25 2015-12-23)[nerv] info: create layer: combinerL1 +(18:23:25 2015-12-23)[nerv] info: create layer: selectL1 +(18:23:25 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(18:23:25 2015-12-23)[nerv] info: create layer: outputL +(18:23:25 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(18:23:25 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(18:23:25 2015-12-23)[nerv] info: create layer: softmaxL +(18:23:25 2015-12-23)[nerv] info: create layer: recurrentL1 +(18:23:25 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(18:23:25 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER28 LR0.01679616]: preparing layers end. +[SCHEDULER ITER28 LR0.01679616]: Generate and initing TNN ... +(18:23:25 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -7216,253 +8199,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER28 LR0.1296]: Initing TNN end. -===ITERATION 28 LR 0.129600=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER28 LR0.1296]: 40092 words processed Wed Nov 18 12:19:40 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.957648. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47576 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92751 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48408 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10714 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12083 clock time -[SCHEDULER ITER28 LR0.1296]: 80099 words processed Wed Nov 18 12:19:52 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.954866. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47325 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90973 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57823 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09662 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11984 clock time -[SCHEDULER ITER28 LR0.1296]: 120004 words processed Wed Nov 18 12:20:04 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.949844. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47062 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91116 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58075 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09554 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12015 clock time -[SCHEDULER ITER28 LR0.1296]: 160114 words processed Wed Nov 18 12:20:16 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.950754. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47209 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92291 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46389 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59603 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10384 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12023 clock time -[SCHEDULER ITER28 LR0.1296]: 200066 words processed Wed Nov 18 12:20:28 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.949861. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46696 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90745 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41420 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57781 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09484 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11963 clock time -[SCHEDULER ITER28 LR0.1296]: 240045 words processed Wed Nov 18 12:20:40 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.945955. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46724 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91063 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42199 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57837 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09877 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11965 clock time -[SCHEDULER ITER28 LR0.1296]: 280057 words processed Wed Nov 18 12:20:52 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.942211. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47004 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92566 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47240 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59893 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10453 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12074 clock time -[SCHEDULER ITER28 LR0.1296]: 320106 words processed Wed Nov 18 12:21:04 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.940331. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46608 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91992 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46156 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59815 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10482 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12048 clock time -[SCHEDULER ITER28 LR0.1296]: 360024 words processed Wed Nov 18 12:21:16 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.938572. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46452 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91680 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43848 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58455 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10033 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12026 clock time -[SCHEDULER ITER28 LR0.1296]: 400089 words processed Wed Nov 18 12:21:28 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.935784. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46687 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92291 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45548 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59456 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10075 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12045 clock time -[SCHEDULER ITER28 LR0.1296]: 440067 words processed Wed Nov 18 12:21:40 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.936314. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46289 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90890 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41067 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57867 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09343 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11991 clock time -[SCHEDULER ITER28 LR0.1296]: 480051 words processed Wed Nov 18 12:21:52 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.936596. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46571 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91960 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43438 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58436 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09648 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12062 clock time -[SCHEDULER ITER28 LR0.1296]: 520140 words processed Wed Nov 18 12:22:04 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.936091. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46598 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92477 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46580 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10321 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12050 clock time -[SCHEDULER ITER28 LR0.1296]: 560132 words processed Wed Nov 18 12:22:16 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.935414. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46406 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91386 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42287 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58180 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09615 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11997 clock time -[SCHEDULER ITER28 LR0.1296]: 600118 words processed Wed Nov 18 12:22:28 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.933190. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46526 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92007 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44464 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58817 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10068 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12076 clock time -[SCHEDULER ITER28 LR0.1296]: 640090 words processed Wed Nov 18 12:22:40 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.931133. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46501 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91867 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43812 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58846 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09634 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12051 clock time -[SCHEDULER ITER28 LR0.1296]: 680075 words processed Wed Nov 18 12:22:52 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.929866. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46311 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91126 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41976 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58215 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09589 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12012 clock time -[SCHEDULER ITER28 LR0.1296]: 720043 words processed Wed Nov 18 12:23:04 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.928898. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46461 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92253 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45457 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59027 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10466 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12074 clock time -[SCHEDULER ITER28 LR0.1296]: 760012 words processed Wed Nov 18 12:23:16 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.927030. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46419 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91529 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43136 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58456 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09667 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12037 clock time -[SCHEDULER ITER28 LR0.1296]: 800113 words processed Wed Nov 18 12:23:28 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.926926. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47535 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96992 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.57577 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62856 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11865 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12383 clock time -[SCHEDULER ITER28 LR0.1296]: 840089 words processed Wed Nov 18 12:23:40 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.925697. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47639 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96560 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.55302 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61785 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11230 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12394 clock time -[SCHEDULER ITER28 LR0.1296]: 880052 words processed Wed Nov 18 12:23:52 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-1.925283. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47514 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.97510 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.58064 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62173 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12226 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12532 clock time +(18:23:25 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:23:25 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:23:25 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:23:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:23:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:23:25 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER28 LR0.01679616]: Initing TNN end. +===ITERATION 28 LR 0.016796=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:23:25 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:23:25 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:23:25 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:23:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:23:25 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:23:25 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:23:25 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER28 LR0.01679616]: 40092 words processed Wed Dec 23 18:23:41 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.937593. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95813 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75089 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77591 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11452 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14876 clock time +[SCHEDULER ITER28 LR0.01679616]: 80099 words processed Wed Dec 23 18:23:57 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.932579. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47970 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92686 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66253 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74364 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09828 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14673 clock time +[SCHEDULER ITER28 LR0.01679616]: 120004 words processed Wed Dec 23 18:24:13 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.927283. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47787 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93047 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67839 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74837 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10134 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14739 clock time +[SCHEDULER ITER28 LR0.01679616]: 160114 words processed Wed Dec 23 18:24:29 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.927605. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47775 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94669 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71401 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76477 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10758 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14760 clock time +[SCHEDULER ITER28 LR0.01679616]: 200066 words processed Wed Dec 23 18:24:45 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.927050. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47343 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92798 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66061 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74474 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09808 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14700 clock time +[SCHEDULER ITER28 LR0.01679616]: 240045 words processed Wed Dec 23 18:25:01 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.923305. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47356 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92937 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66584 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74577 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10116 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14691 clock time +[SCHEDULER ITER28 LR0.01679616]: 280057 words processed Wed Dec 23 18:25:17 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.919053. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47539 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94789 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71611 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76602 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10617 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14764 clock time +[SCHEDULER ITER28 LR0.01679616]: 320106 words processed Wed Dec 23 18:25:33 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.917004. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47254 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94743 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71409 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76636 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10716 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14770 clock time +[SCHEDULER ITER28 LR0.01679616]: 360024 words processed Wed Dec 23 18:25:49 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.915434. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46956 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93144 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66625 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74744 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09875 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14689 clock time +[SCHEDULER ITER28 LR0.01679616]: 400089 words processed Wed Dec 23 18:26:05 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.912760. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47230 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94954 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71494 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76797 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10559 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14785 clock time +[SCHEDULER ITER28 LR0.01679616]: 440067 words processed Wed Dec 23 18:26:21 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.913261. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47594 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.98504 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.76990 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77618 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10861 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14951 clock time +[SCHEDULER ITER28 LR0.01679616]: 480051 words processed Wed Dec 23 18:26:37 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.913769. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47377 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96849 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73295 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76551 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10448 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14943 clock time +[SCHEDULER ITER28 LR0.01679616]: 520140 words processed Wed Dec 23 18:26:53 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.913227. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47447 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.97465 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.75890 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77730 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10982 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14852 clock time +[SCHEDULER ITER28 LR0.01679616]: 560132 words processed Wed Dec 23 18:27:09 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.912838. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46928 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93075 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66519 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75226 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09731 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14733 clock time +[SCHEDULER ITER28 LR0.01679616]: 600118 words processed Wed Dec 23 18:27:25 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.910651. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92714 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65665 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74572 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09861 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14691 clock time +[SCHEDULER ITER28 LR0.01679616]: 640090 words processed Wed Dec 23 18:27:41 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.908467. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46877 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92746 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65454 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74448 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09841 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14715 clock time +[SCHEDULER ITER28 LR0.01679616]: 680075 words processed Wed Dec 23 18:27:57 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.907243. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46891 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92822 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65727 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74515 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10029 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14684 clock time +[SCHEDULER ITER28 LR0.01679616]: 720043 words processed Wed Dec 23 18:28:13 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.905973. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93389 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67533 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75342 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10022 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14717 clock time +[SCHEDULER ITER28 LR0.01679616]: 760012 words processed Wed Dec 23 18:28:29 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.903936. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46861 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92752 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65785 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74695 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09804 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14695 clock time +[SCHEDULER ITER28 LR0.01679616]: 800113 words processed Wed Dec 23 18:28:45 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.903564. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46992 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94952 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76666 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10531 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14759 clock time +[SCHEDULER ITER28 LR0.01679616]: 840089 words processed Wed Dec 23 18:29:01 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.902237. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93182 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66070 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74620 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09842 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14718 clock time +[SCHEDULER ITER28 LR0.01679616]: 880052 words processed Wed Dec 23 18:29:17 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-1.901688. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46868 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92856 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65997 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74575 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09935 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14740 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER28 LR0.1296]: Displaying result: -[SCHEDULER ITER28 LR0.1296]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 92.670119900574> <PPL_OOV 84.17526942258> <LOGP -1789630.3482288> -[SCHEDULER ITER28 LR0.1296]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER28 LR0.1296]: shuffling training file +(18:29:20 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER28 LR0.01679616]: Displaying result: +[SCHEDULER ITER28 LR0.01679616]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 87.679293915198> <PPL_OOV 79.733828126534> <LOGP -1767746.0574361> +[SCHEDULER ITER28 LR0.01679616]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER28 LR0.01679616]: shuffling training file ===PEEK ON TEST 28=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER28 LR0.1296]: 40087 words processed Wed Nov 18 12:24:01 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-2.120814. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47127 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92295 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62049 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11555 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:29:22 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:29:22 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:29:22 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:29:22 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:29:22 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:29:22 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:29:22 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER28 LR0.01679616]: 40087 words processed Wed Dec 23 18:29:30 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-2.107300. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49172 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12355 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.87686 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14367 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER28 LR0.1296]: Displaying result: -[SCHEDULER ITER28 LR0.1296]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 152.54022747212> <PPL_OOV 132.83640444882> <LOGP -175025.0295268> -[SCHEDULER ITER28 LR0.1296]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(18:29:38 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER28 LR0.01679616]: Displaying result: +[SCHEDULER ITER28 LR0.01679616]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.82266563629> <PPL_OOV 128.37406510027> <LOGP -173801.78332913> +[SCHEDULER ITER28 LR0.01679616]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 28=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER28 LR0.1296]: 40095 words processed Wed Nov 18 12:24:10 2015. - [SCHEDULER ITER28 LR0.1296]: log prob per sample :-2.196983. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47218 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93150 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.63056 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11558 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:29:38 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:29:38 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:29:38 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:29:38 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:29:38 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:29:38 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:29:38 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER28 LR0.01679616]: 40095 words processed Wed Dec 23 18:29:46 2015. + [SCHEDULER ITER28 LR0.01679616]: log prob per sample :-2.181272. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49170 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.16884 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92027 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14201 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER28 LR0.1296]: Displaying result: -[SCHEDULER ITER28 LR0.1296]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 156.97759103608> <PPL_OOV 140.13454203292> <LOGP -158329.17384608> -[SCHEDULER ITER28 LR0.1296]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER28 LR0.1296]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.28... - -[SCHEDULER ITER29 LR0.07776]: preparing parameters... -[SCHEDULER ITER29 LR0.07776]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.28... +(18:29:52 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER28 LR0.01679616]: Displaying result: +[SCHEDULER ITER28 LR0.01679616]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 151.32539158313> <PPL_OOV 135.52706623444> <LOGP -157258.24050541> +[SCHEDULER ITER28 LR0.01679616]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER28 LR0.01679616]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.28... + +[SCHEDULER ITER29 LR0.010077696]: preparing parameters... +[SCHEDULER ITER29 LR0.010077696]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.28... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 1 from 1062789 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 1066441 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 35823808 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 35940048 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 5 from 70466632 -[SCHEDULER ITER29 LR0.07776]: preparing parameters end. -[SCHEDULER ITER29 LR0.07776]: preparing layers... -(12:24:19 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:24:19 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:24:19 2015-11-18)[nerv] info: create layer: combinerL1 -(12:24:19 2015-11-18)[nerv] info: create layer: outputL -(12:24:19 2015-11-18)[nerv] info: create layer: softmaxL -(12:24:19 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER29 LR0.07776]: preparing layers end. -[SCHEDULER ITER29 LR0.07776]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3663 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 119931 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1183060 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35917808 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70454708 +[SCHEDULER ITER29 LR0.010077696]: preparing parameters end. +[SCHEDULER ITER29 LR0.010077696]: preparing layers... +(18:29:58 2015-12-23)[nerv] info: create layer: sigmoidL1 +(18:29:58 2015-12-23)[nerv] info: create layer: combinerL1 +(18:29:58 2015-12-23)[nerv] info: create layer: selectL1 +(18:29:58 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(18:29:58 2015-12-23)[nerv] info: create layer: outputL +(18:29:58 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(18:29:58 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(18:29:58 2015-12-23)[nerv] info: create layer: softmaxL +(18:29:58 2015-12-23)[nerv] info: create layer: recurrentL1 +(18:29:58 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(18:29:58 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER29 LR0.010077696]: preparing layers end. +[SCHEDULER ITER29 LR0.010077696]: Generate and initing TNN ... +(18:29:58 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -7478,253 +8497,289 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER29 LR0.07776]: Initing TNN end. -===ITERATION 29 LR 0.077760=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER29 LR0.07776]: 40092 words processed Wed Nov 18 12:24:30 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.946777. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47512 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93269 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.50526 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61761 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10694 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12063 clock time -[SCHEDULER ITER29 LR0.07776]: 80099 words processed Wed Nov 18 12:24:42 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.943867. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90926 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43729 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59207 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09450 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11983 clock time -[SCHEDULER ITER29 LR0.07776]: 120004 words processed Wed Nov 18 12:24:54 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.939040. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91327 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44627 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59133 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09606 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11977 clock time -[SCHEDULER ITER29 LR0.07776]: 160114 words processed Wed Nov 18 12:25:06 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.940226. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47343 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92803 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.48535 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61050 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10159 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12071 clock time -[SCHEDULER ITER29 LR0.07776]: 200066 words processed Wed Nov 18 12:25:18 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.939366. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47098 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92476 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46657 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59858 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09890 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12052 clock time -[SCHEDULER ITER29 LR0.07776]: 240045 words processed Wed Nov 18 12:25:30 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.935511. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91054 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42877 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58785 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09333 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11971 clock time -[SCHEDULER ITER29 LR0.07776]: 280057 words processed Wed Nov 18 12:25:42 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.931795. +(18:29:58 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:29:58 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:29:58 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:29:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:29:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:29:58 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER29 LR0.010077696]: Initing TNN end. +===ITERATION 29 LR 0.010078=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:29:58 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:29:58 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:29:58 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:29:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:29:58 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:29:58 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:29:58 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER29 LR0.010077696]: 40092 words processed Wed Dec 23 18:30:14 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.935089. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48443 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.99168 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.80028 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11633 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14970 clock time +[SCHEDULER ITER29 LR0.010077696]: 80099 words processed Wed Dec 23 18:30:30 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.930141. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47957 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93188 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67501 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74348 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10319 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14712 clock time +[SCHEDULER ITER29 LR0.010077696]: 120004 words processed Wed Dec 23 18:30:46 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.924892. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47855 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92836 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66481 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73826 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09934 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14684 clock time +[SCHEDULER ITER29 LR0.010077696]: 160114 words processed Wed Dec 23 18:31:02 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.925254. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47827 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95050 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76038 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10787 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14752 clock time +[SCHEDULER ITER29 LR0.010077696]: 200066 words processed Wed Dec 23 18:31:18 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.924738. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92923 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73802 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10122 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14697 clock time +[SCHEDULER ITER29 LR0.010077696]: 240045 words processed Wed Dec 23 18:31:34 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.921014. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47360 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92934 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73945 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09923 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14690 clock time +[SCHEDULER ITER29 LR0.010077696]: 280057 words processed Wed Dec 23 18:31:50 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.916791. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47548 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94966 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76091 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10704 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14747 clock time +[SCHEDULER ITER29 LR0.010077696]: 320106 words processed Wed Dec 23 18:32:06 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.914754. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48170 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00080 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.81516 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.78503 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11659 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.15135 clock time +[SCHEDULER ITER29 LR0.010077696]: 360024 words processed Wed Dec 23 18:32:22 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.913190. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47034 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92887 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65859 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73933 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09979 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14733 clock time +[SCHEDULER ITER29 LR0.010077696]: 400089 words processed Wed Dec 23 18:32:38 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.910529. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47171 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95014 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70830 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75901 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10767 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14745 clock time +[SCHEDULER ITER29 LR0.010077696]: 440067 words processed Wed Dec 23 18:32:54 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.911036. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96967 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72769 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75654 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10489 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14883 clock time +[SCHEDULER ITER29 LR0.010077696]: 480051 words processed Wed Dec 23 18:33:10 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.911543. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47372 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96559 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75710 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10563 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14971 clock time +[SCHEDULER ITER29 LR0.010077696]: 520140 words processed Wed Dec 23 18:33:26 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.911001. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47162 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95105 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70828 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75903 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10610 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14738 clock time +[SCHEDULER ITER29 LR0.010077696]: 560132 words processed Wed Dec 23 18:33:42 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.910638. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47199 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93876 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67315 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74594 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09915 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14775 clock time +[SCHEDULER ITER29 LR0.010077696]: 600118 words processed Wed Dec 23 18:33:58 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.908460. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46924 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92980 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65610 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10030 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14668 clock time +[SCHEDULER ITER29 LR0.010077696]: 640090 words processed Wed Dec 23 18:34:14 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.906268. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46863 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93503 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66677 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74399 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10131 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14756 clock time +[SCHEDULER ITER29 LR0.010077696]: 680075 words processed Wed Dec 23 18:34:30 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.905046. [global_conf.timer]: time spent on tnn_beforeprocess:0.46920 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91781 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46106 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60264 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09941 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12016 clock time -[SCHEDULER ITER29 LR0.07776]: 320106 words processed Wed Nov 18 12:25:54 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.929898. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46857 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92084 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46699 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60428 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10031 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12045 clock time -[SCHEDULER ITER29 LR0.07776]: 360024 words processed Wed Nov 18 12:26:06 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.928153. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46496 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91159 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42890 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58868 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09209 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11988 clock time -[SCHEDULER ITER29 LR0.07776]: 400089 words processed Wed Nov 18 12:26:18 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.925337. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46626 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92443 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60646 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09985 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12053 clock time -[SCHEDULER ITER29 LR0.07776]: 440067 words processed Wed Nov 18 12:26:30 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.925902. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46441 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91349 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43133 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59089 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09387 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11978 clock time -[SCHEDULER ITER29 LR0.07776]: 480051 words processed Wed Nov 18 12:26:42 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.926173. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46622 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92150 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44978 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59630 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09655 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12026 clock time -[SCHEDULER ITER29 LR0.07776]: 520140 words processed Wed Nov 18 12:26:54 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.925626. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46571 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92512 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47393 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60903 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10114 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12035 clock time -[SCHEDULER ITER29 LR0.07776]: 560132 words processed Wed Nov 18 12:27:06 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.924926. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46577 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91608 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43582 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59094 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09418 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12049 clock time -[SCHEDULER ITER29 LR0.07776]: 600118 words processed Wed Nov 18 12:27:18 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.922706. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46470 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91317 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43105 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59057 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09363 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12010 clock time -[SCHEDULER ITER29 LR0.07776]: 640090 words processed Wed Nov 18 12:27:30 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.920601. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46423 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91189 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42763 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59020 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09271 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12012 clock time -[SCHEDULER ITER29 LR0.07776]: 680075 words processed Wed Nov 18 12:27:42 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.919309. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46435 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91007 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41862 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58598 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09165 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11954 clock time -[SCHEDULER ITER29 LR0.07776]: 720043 words processed Wed Nov 18 12:27:54 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.918272. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91020 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42724 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09533 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11972 clock time -[SCHEDULER ITER29 LR0.07776]: 760012 words processed Wed Nov 18 12:28:06 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.916346. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46430 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91184 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43111 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59128 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09399 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11987 clock time -[SCHEDULER ITER29 LR0.07776]: 800113 words processed Wed Nov 18 12:28:18 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.916197. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46660 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92285 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46968 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60634 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10210 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12018 clock time -[SCHEDULER ITER29 LR0.07776]: 840089 words processed Wed Nov 18 12:28:30 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.914943. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46473 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91444 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43742 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59262 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09682 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11978 clock time -[SCHEDULER ITER29 LR0.07776]: 880052 words processed Wed Nov 18 12:28:42 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-1.914477. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46559 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91155 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43238 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59180 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09397 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11945 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92685 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.64803 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73844 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09807 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14683 clock time +[SCHEDULER ITER29 LR0.010077696]: 720043 words processed Wed Dec 23 18:34:46 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.903766. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46941 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93326 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66093 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74052 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09956 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14725 clock time +[SCHEDULER ITER29 LR0.010077696]: 760012 words processed Wed Dec 23 18:35:02 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.901729. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46929 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93347 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66150 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74070 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09871 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14745 clock time +[SCHEDULER ITER29 LR0.010077696]: 800113 words processed Wed Dec 23 18:35:18 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.901360. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47156 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95019 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.70924 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76113 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10613 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14729 clock time +[SCHEDULER ITER29 LR0.010077696]: 840089 words processed Wed Dec 23 18:35:34 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.900022. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46906 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92771 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65071 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73869 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09877 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14694 clock time +[SCHEDULER ITER29 LR0.010077696]: 880052 words processed Wed Dec 23 18:35:50 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-1.899465. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46943 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.92874 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65623 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.73954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09986 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14729 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER29 LR0.07776]: Displaying result: -[SCHEDULER ITER29 LR0.07776]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 90.331762955738> <PPL_OOV 82.103047796228> <LOGP -1779567.3279209> -[SCHEDULER ITER29 LR0.07776]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER29 LR0.07776]: shuffling training file +(18:35:53 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER29 LR0.010077696]: Displaying result: +[SCHEDULER ITER29 LR0.010077696]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 87.221774717147> <PPL_OOV 79.326864332909> <LOGP -1765680.2057695> +[SCHEDULER ITER29 LR0.010077696]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER29 LR0.010077696]: shuffling training file ===PEEK ON TEST 29=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER29 LR0.07776]: 40087 words processed Wed Nov 18 12:28:51 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-2.115230. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47088 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92767 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62634 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11572 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:35:55 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:35:55 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:35:55 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:35:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:35:55 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:35:55 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:35:55 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER29 LR0.010077696]: 40087 words processed Wed Dec 23 18:36:03 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-2.106531. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49209 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12100 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.87571 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14415 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER29 LR0.07776]: Displaying result: -[SCHEDULER ITER29 LR0.07776]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 150.61305621631> <PPL_OOV 131.20309244406> <LOGP -174582.1298087> -[SCHEDULER ITER29 LR0.07776]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(18:36:11 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER29 LR0.010077696]: Displaying result: +[SCHEDULER ITER29 LR0.010077696]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.4677679748> <PPL_OOV 128.13045842904> <LOGP -173733.78568185> +[SCHEDULER ITER29 LR0.010077696]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 29=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER29 LR0.07776]: 40095 words processed Wed Nov 18 12:29:00 2015. - [SCHEDULER ITER29 LR0.07776]: log prob per sample :-2.191859. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47179 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92795 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62741 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11570 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:36:11 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:36:11 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:36:11 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:36:11 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:36:11 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:36:11 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:36:11 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER29 LR0.010077696]: 40095 words processed Wed Dec 23 18:36:19 2015. + [SCHEDULER ITER29 LR0.010077696]: log prob per sample :-2.180385. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49140 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.17687 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.92939 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14258 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER29 LR0.07776]: Displaying result: -[SCHEDULER ITER29 LR0.07776]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 155.05059609032> <PPL_OOV 138.44697733307> <LOGP -157941.06936514> -[SCHEDULER ITER29 LR0.07776]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER29 LR0.07776]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.29... - -[SCHEDULER ITER30 LR0.046656]: preparing parameters... -[SCHEDULER ITER30 LR0.046656]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.29... +(18:36:25 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER29 LR0.010077696]: Displaying result: +[SCHEDULER ITER29 LR0.010077696]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 150.90326752234> <PPL_OOV 135.20720715679> <LOGP -157182.54835229> +[SCHEDULER ITER29 LR0.010077696]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER29 LR0.010077696]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.29... + +[SCHEDULER ITER30 LR0.0060466176]: preparing parameters... +[SCHEDULER ITER30 LR0.0060466176]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.29... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 1 from 34526461 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 34530113 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 69287366 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 69403593 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 5 from 70466227 -[SCHEDULER ITER30 LR0.046656]: preparing parameters end. -[SCHEDULER ITER30 LR0.046656]: preparing layers... -(12:29:09 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:29:09 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:29:09 2015-11-18)[nerv] info: create layer: combinerL1 -(12:29:09 2015-11-18)[nerv] info: create layer: outputL -(12:29:09 2015-11-18)[nerv] info: create layer: softmaxL -(12:29:09 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER30 LR0.046656]: preparing layers end. -[SCHEDULER ITER30 LR0.046656]: Generate and initing TNN ... +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} + +reading chunk 1 from 3662 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} + +reading chunk 2 from 119933 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} + +reading chunk 3 from 1183012 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} + +reading chunk 4 from 35917591 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} + +reading chunk 5 from 70454478 +[SCHEDULER ITER30 LR0.0060466176]: preparing parameters end. +[SCHEDULER ITER30 LR0.0060466176]: preparing layers... +(18:36:31 2015-12-23)[nerv] info: create layer: sigmoidL1 +(18:36:31 2015-12-23)[nerv] info: create layer: combinerL1 +(18:36:31 2015-12-23)[nerv] info: create layer: selectL1 +(18:36:31 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(18:36:31 2015-12-23)[nerv] info: create layer: outputL +(18:36:31 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(18:36:31 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(18:36:31 2015-12-23)[nerv] info: create layer: softmaxL +(18:36:31 2015-12-23)[nerv] info: create layer: recurrentL1 +(18:36:31 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(18:36:31 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. +[SCHEDULER ITER30 LR0.0060466176]: preparing layers end. +[SCHEDULER ITER30 LR0.0060466176]: Generate and initing TNN ... +(18:36:31 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -7740,1602 +8795,324 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER30 LR0.046656]: Initing TNN end. -===ITERATION 30 LR 0.046656=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER30 LR0.046656]: 40092 words processed Wed Nov 18 12:29:20 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.939564. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47323 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91388 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46581 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60149 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10874 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11928 clock time -[SCHEDULER ITER30 LR0.046656]: 80099 words processed Wed Nov 18 12:29:32 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.936511. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47190 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89903 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41530 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58109 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09897 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11891 clock time -[SCHEDULER ITER30 LR0.046656]: 120004 words processed Wed Nov 18 12:29:44 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.931733. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47171 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90058 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43192 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58581 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10386 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11913 clock time -[SCHEDULER ITER30 LR0.046656]: 160114 words processed Wed Nov 18 12:29:56 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.933054. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47051 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90785 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44202 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59732 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09959 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11939 clock time -[SCHEDULER ITER30 LR0.046656]: 200066 words processed Wed Nov 18 12:30:08 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.932265. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46658 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90114 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40893 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57712 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09861 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11913 clock time -[SCHEDULER ITER30 LR0.046656]: 240045 words processed Wed Nov 18 12:30:20 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.928476. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46807 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91218 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42615 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58220 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09673 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11965 clock time -[SCHEDULER ITER30 LR0.046656]: 280057 words processed Wed Nov 18 12:30:32 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.924786. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46728 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90785 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59291 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10576 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11904 clock time -[SCHEDULER ITER30 LR0.046656]: 320106 words processed Wed Nov 18 12:30:44 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.922894. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46756 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91692 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46827 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60127 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11160 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11938 clock time -[SCHEDULER ITER30 LR0.046656]: 360024 words processed Wed Nov 18 12:30:56 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.921186. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46577 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90688 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42380 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58556 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09562 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11943 clock time -[SCHEDULER ITER30 LR0.046656]: 400089 words processed Wed Nov 18 12:31:08 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.918359. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46498 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91982 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46522 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60111 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10899 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11999 clock time -[SCHEDULER ITER30 LR0.046656]: 440067 words processed Wed Nov 18 12:31:20 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.918965. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46571 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91087 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44413 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58651 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11306 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11988 clock time -[SCHEDULER ITER30 LR0.046656]: 480051 words processed Wed Nov 18 12:31:32 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.919242. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46365 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90509 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.40934 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58151 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09419 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11921 clock time -[SCHEDULER ITER30 LR0.046656]: 520140 words processed Wed Nov 18 12:31:44 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.918685. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46596 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92273 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46873 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59898 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11193 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11941 clock time -[SCHEDULER ITER30 LR0.046656]: 560132 words processed Wed Nov 18 12:31:56 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.918001. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46366 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91923 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43742 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58558 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10122 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12021 clock time -[SCHEDULER ITER30 LR0.046656]: 600118 words processed Wed Nov 18 12:32:08 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.915795. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46320 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90927 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41858 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58082 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09909 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11923 clock time -[SCHEDULER ITER30 LR0.046656]: 640090 words processed Wed Nov 18 12:32:20 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.913679. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46256 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90133 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42019 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58192 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10900 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11919 clock time -[SCHEDULER ITER30 LR0.046656]: 680075 words processed Wed Nov 18 12:32:32 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.912380. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46082 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89145 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.38421 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57646 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09318 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11829 clock time -[SCHEDULER ITER30 LR0.046656]: 720043 words processed Wed Nov 18 12:32:44 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.911315. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46067 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89267 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.39630 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.57730 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10051 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11866 clock time -[SCHEDULER ITER30 LR0.046656]: 760012 words processed Wed Nov 18 12:32:56 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.909373. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46430 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90617 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41359 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58324 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09288 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11921 clock time -[SCHEDULER ITER30 LR0.046656]: 800113 words processed Wed Nov 18 12:33:08 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.909204. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46693 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92838 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.47629 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60115 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10846 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12048 clock time -[SCHEDULER ITER30 LR0.046656]: 840089 words processed Wed Nov 18 12:33:20 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.907946. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46535 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92198 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45822 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58926 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11251 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12044 clock time -[SCHEDULER ITER30 LR0.046656]: 880052 words processed Wed Nov 18 12:33:32 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-1.907455. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46282 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91155 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42637 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58580 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09735 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11968 clock time +(18:36:31 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:36:31 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:36:31 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:36:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:36:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:36:31 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +[SCHEDULER ITER30 LR0.0060466176]: Initing TNN end. +===ITERATION 30 LR 0.006047=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf... +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:36:31 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:36:31 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:36:31 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:36:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:36:31 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:36:31 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:36:31 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER30 LR0.0060466176]: 40092 words processed Wed Dec 23 18:36:47 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.933473. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48601 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.00148 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.81806 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.79077 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11517 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14970 clock time +[SCHEDULER ITER30 LR0.0060466176]: 80099 words processed Wed Dec 23 18:37:03 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.928598. + [global_conf.timer]: time spent on tnn_beforeprocess:0.48009 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93287 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67835 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75450 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09618 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14736 clock time +[SCHEDULER ITER30 LR0.0060466176]: 120004 words processed Wed Dec 23 18:37:19 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.923346. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47777 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93206 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67283 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74826 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09641 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14704 clock time +[SCHEDULER ITER30 LR0.0060466176]: 160114 words processed Wed Dec 23 18:37:35 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.923724. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47819 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95180 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72149 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76992 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10499 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14738 clock time +[SCHEDULER ITER30 LR0.0060466176]: 200066 words processed Wed Dec 23 18:37:51 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.923250. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47426 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93183 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66471 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74733 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09615 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14682 clock time +[SCHEDULER ITER30 LR0.0060466176]: 240045 words processed Wed Dec 23 18:38:07 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.919531. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47467 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93634 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67711 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75214 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09826 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14744 clock time +[SCHEDULER ITER30 LR0.0060466176]: 280057 words processed Wed Dec 23 18:38:23 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.915335. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47605 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95330 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72368 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76993 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10445 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14746 clock time +[SCHEDULER ITER30 LR0.0060466176]: 320106 words processed Wed Dec 23 18:38:39 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.913307. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47387 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95223 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71729 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76824 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10381 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14736 clock time +[SCHEDULER ITER30 LR0.0060466176]: 360024 words processed Wed Dec 23 18:38:55 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.911746. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46964 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93207 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66315 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74756 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09657 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14677 clock time +[SCHEDULER ITER30 LR0.0060466176]: 400089 words processed Wed Dec 23 18:39:11 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.909096. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47320 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.96087 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.73607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10722 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14819 clock time +[SCHEDULER ITER30 LR0.0060466176]: 440067 words processed Wed Dec 23 18:39:27 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.909611. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93236 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66408 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74960 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09669 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14731 clock time +[SCHEDULER ITER30 LR0.0060466176]: 480051 words processed Wed Dec 23 18:39:43 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.910123. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46966 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93197 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65965 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74827 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09652 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14694 clock time +[SCHEDULER ITER30 LR0.0060466176]: 520140 words processed Wed Dec 23 18:39:59 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.909583. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47240 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95775 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.72287 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.77050 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10348 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14728 clock time +[SCHEDULER ITER30 LR0.0060466176]: 560132 words processed Wed Dec 23 18:40:15 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.909230. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46968 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93303 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66573 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75125 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09756 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14732 clock time +[SCHEDULER ITER30 LR0.0060466176]: 600118 words processed Wed Dec 23 18:40:31 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.907063. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94742 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.69416 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75986 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09811 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14735 clock time +[SCHEDULER ITER30 LR0.0060466176]: 640090 words processed Wed Dec 23 18:40:47 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.904868. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46939 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93292 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65911 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74826 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09483 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14653 clock time +[SCHEDULER ITER30 LR0.0060466176]: 680075 words processed Wed Dec 23 18:41:03 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.903645. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46929 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93361 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.66025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74886 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09470 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14685 clock time +[SCHEDULER ITER30 LR0.0060466176]: 720043 words processed Wed Dec 23 18:41:19 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.902356. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46885 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93812 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67181 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75214 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09676 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14694 clock time +[SCHEDULER ITER30 LR0.0060466176]: 760012 words processed Wed Dec 23 18:41:35 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.900318. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46863 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.93204 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.65915 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.74851 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09415 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14700 clock time +[SCHEDULER ITER30 LR0.0060466176]: 800113 words processed Wed Dec 23 18:41:51 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.899956. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47088 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95391 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76849 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10212 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14733 clock time +[SCHEDULER ITER30 LR0.0060466176]: 840089 words processed Wed Dec 23 18:42:07 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.898611. + [global_conf.timer]: time spent on tnn_beforeprocess:0.47058 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.95969 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.71476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.76569 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10063 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14783 clock time +[SCHEDULER ITER30 LR0.0060466176]: 880052 words processed Wed Dec 23 18:42:23 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-1.898048. + [global_conf.timer]: time spent on tnn_beforeprocess:0.46919 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:5.94141 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:14.67982 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.75334 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09835 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14721 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER30 LR0.046656]: Displaying result: -[SCHEDULER ITER30 LR0.046656]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 88.846002225269> <PPL_OOV 80.784476409319> <LOGP -1773031.0487294> -[SCHEDULER ITER30 LR0.046656]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER30 LR0.046656]: shuffling training file +(18:42:26 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER30 LR0.0060466176]: Displaying result: +[SCHEDULER ITER30 LR0.0060466176]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.931315916909> <PPL_OOV 79.068578625117> <LOGP -1764363.5769401> +[SCHEDULER ITER30 LR0.0060466176]: Doing on /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/train_fn_shuf end. +[SCHEDULER ITER30 LR0.0060466176]: shuffling training file ===PEEK ON TEST 30=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER30 LR0.046656]: 40087 words processed Wed Nov 18 12:33:41 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-2.111073. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47174 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92856 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.62710 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11577 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:42:28 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:42:28 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:42:28 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:42:28 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:42:28 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:42:28 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:42:28 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER30 LR0.0060466176]: 40087 words processed Wed Dec 23 18:42:36 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-2.106035. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49186 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.11777 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.87213 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14333 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER30 LR0.046656]: Displaying result: -[SCHEDULER ITER30 LR0.046656]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 149.06832367442> <PPL_OOV 130.00664401787> <LOGP -174254.18008769> -[SCHEDULER ITER30 LR0.046656]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +(18:42:44 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER30 LR0.0060466176]: Displaying result: +[SCHEDULER ITER30 LR0.0060466176]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.23750893173> <PPL_OOV 127.96908120768> <LOGP -173688.66942091> +[SCHEDULER ITER30 LR0.0060466176]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 30=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER30 LR0.046656]: 40095 words processed Wed Nov 18 12:33:50 2015. - [SCHEDULER ITER30 LR0.046656]: log prob per sample :-2.187932. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47040 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92161 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.61760 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11538 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:42:44 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:42:44 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:42:44 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:42:44 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:42:44 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:42:44 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:42:44 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER ITER30 LR0.0060466176]: 40095 words processed Wed Dec 23 18:42:52 2015. + [SCHEDULER ITER30 LR0.0060466176]: log prob per sample :-2.179798. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49197 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.16260 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.91481 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14253 clock time [LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER30 LR0.046656]: Displaying result: -[SCHEDULER ITER30 LR0.046656]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 153.4615968442> <PPL_OOV 137.16123257402> <LOGP -157642.18652739> -[SCHEDULER ITER30 LR0.046656]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER30 LR0.046656]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.30... - -[SCHEDULER ITER31 LR0.0279936]: preparing parameters... -[SCHEDULER ITER31 LR0.0279936]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.30... -reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 1 from 1062556 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 1066208 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 35823367 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 35939586 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 5 from 70465956 -[SCHEDULER ITER31 LR0.0279936]: preparing parameters end. -[SCHEDULER ITER31 LR0.0279936]: preparing layers... -(12:33:59 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:33:59 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:33:59 2015-11-18)[nerv] info: create layer: combinerL1 -(12:33:59 2015-11-18)[nerv] info: create layer: outputL -(12:33:59 2015-11-18)[nerv] info: create layer: softmaxL -(12:33:59 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER31 LR0.0279936]: preparing layers end. -[SCHEDULER ITER31 LR0.0279936]: Generate and initing TNN ... -<input> selectL1 0 -selectL1 recurrentL1 0 -recurrentL1 sigmoidL1 0 -sigmoidL1 combinerL1 0 -combinerL1 recurrentL1 1 -combinerL1 outputL 0 -outputL softmaxL 0 -<input> softmaxL 0 -softmaxL <output> 0 -recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 -outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 -sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 -softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER31 LR0.0279936]: Initing TNN end. -===ITERATION 31 LR 0.027994=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER31 LR0.0279936]: 40092 words processed Wed Nov 18 12:34:10 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.935039. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47640 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93064 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.52273 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62723 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11320 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12098 clock time -[SCHEDULER ITER31 LR0.0279936]: 80099 words processed Wed Nov 18 12:34:22 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.931902. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47206 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89292 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41120 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09281 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11933 clock time -[SCHEDULER ITER31 LR0.0279936]: 120004 words processed Wed Nov 18 12:34:34 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.927163. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46995 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89755 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58695 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09593 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11970 clock time -[SCHEDULER ITER31 LR0.0279936]: 160114 words processed Wed Nov 18 12:34:46 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.928513. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47165 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90869 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44753 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60222 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09758 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12003 clock time -[SCHEDULER ITER31 LR0.0279936]: 200066 words processed Wed Nov 18 12:34:58 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.927746. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46810 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90199 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42310 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58786 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09741 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11991 clock time -[SCHEDULER ITER31 LR0.0279936]: 240045 words processed Wed Nov 18 12:35:10 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.924005. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46763 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90358 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42622 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59212 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09478 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12022 clock time -[SCHEDULER ITER31 LR0.0279936]: 280057 words processed Wed Nov 18 12:35:22 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.920332. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46890 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91181 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46435 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60724 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10423 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12031 clock time -[SCHEDULER ITER31 LR0.0279936]: 320106 words processed Wed Nov 18 12:35:34 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.918430. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46680 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90794 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44848 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60180 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10136 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12041 clock time -[SCHEDULER ITER31 LR0.0279936]: 360024 words processed Wed Nov 18 12:35:46 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.916720. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46311 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89761 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41214 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58712 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09475 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11972 clock time -[SCHEDULER ITER31 LR0.0279936]: 400089 words processed Wed Nov 18 12:35:58 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.913875. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46481 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90749 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44837 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60297 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10332 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12040 clock time -[SCHEDULER ITER31 LR0.0279936]: 440067 words processed Wed Nov 18 12:36:10 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.914482. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46634 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91175 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44437 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59617 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09966 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12105 clock time -[SCHEDULER ITER31 LR0.0279936]: 480051 words processed Wed Nov 18 12:36:22 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.914748. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46635 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91093 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.43652 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59396 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09694 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12098 clock time -[SCHEDULER ITER31 LR0.0279936]: 520140 words processed Wed Nov 18 12:36:34 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.914169. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46453 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90855 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44925 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10541 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12020 clock time -[SCHEDULER ITER31 LR0.0279936]: 560132 words processed Wed Nov 18 12:36:46 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.913487. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46373 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.89911 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41676 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58876 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09840 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12024 clock time -[SCHEDULER ITER31 LR0.0279936]: 600118 words processed Wed Nov 18 12:36:58 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.911271. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46461 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90178 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42096 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.58807 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09958 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11996 clock time -[SCHEDULER ITER31 LR0.0279936]: 640090 words processed Wed Nov 18 12:37:10 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.909133. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46430 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90341 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.42658 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59187 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09868 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12053 clock time -[SCHEDULER ITER31 LR0.0279936]: 680075 words processed Wed Nov 18 12:37:22 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.907806. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46383 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.90403 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.41658 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59090 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09025 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12035 clock time -[SCHEDULER ITER31 LR0.0279936]: 720043 words processed Wed Nov 18 12:37:34 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.906706. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48339 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00535 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.69570 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67737 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12577 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12706 clock time -[SCHEDULER ITER31 LR0.0279936]: 760012 words processed Wed Nov 18 12:37:46 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.904727. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48566 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.04027 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.77595 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.69803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13485 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13019 clock time -[SCHEDULER ITER31 LR0.0279936]: 800113 words processed Wed Nov 18 12:37:58 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.904523. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47381 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98650 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65570 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12598 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12719 clock time -[SCHEDULER ITER31 LR0.0279936]: 840089 words processed Wed Nov 18 12:38:10 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.903243. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48731 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.04646 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.79695 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.71087 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12804 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13070 clock time -[SCHEDULER ITER31 LR0.0279936]: 880052 words processed Wed Nov 18 12:38:23 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-1.902719. - [global_conf.timer]: time spent on tnn_beforeprocess:0.50049 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.10777 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.98369 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.76807 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.16031 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13588 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER31 LR0.0279936]: Displaying result: -[SCHEDULER ITER31 LR0.0279936]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 87.858674349694> <PPL_OOV 79.906232708186> <LOGP -1768618.0491743> -[SCHEDULER ITER31 LR0.0279936]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER31 LR0.0279936]: shuffling training file -===PEEK ON TEST 31=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER31 LR0.0279936]: 40087 words processed Wed Nov 18 12:38:32 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-2.108291. - [global_conf.timer]: time spent on tnn_beforeprocess:0.49792 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.03650 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.78124 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12340 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER31 LR0.0279936]: Displaying result: -[SCHEDULER ITER31 LR0.0279936]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 147.99858014623> <PPL_OOV 129.20534519493> <LOGP -174032.85002343> -[SCHEDULER ITER31 LR0.0279936]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. -===VALIDATION 31=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER31 LR0.0279936]: 40095 words processed Wed Nov 18 12:38:42 2015. - [SCHEDULER ITER31 LR0.0279936]: log prob per sample :-2.185472. - [global_conf.timer]: time spent on tnn_beforeprocess:0.49615 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.09369 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.84657 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12753 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER31 LR0.0279936]: Displaying result: -[SCHEDULER ITER31 LR0.0279936]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 152.3641825233> <PPL_OOV 136.30130176941> <LOGP -157440.72026995> -[SCHEDULER ITER31 LR0.0279936]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER31 LR0.0279936]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.31... - -[SCHEDULER ITER32 LR0.01679616]: preparing parameters... -[SCHEDULER ITER32 LR0.01679616]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.31... -reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 1 from 34526275 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 34529927 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 69286931 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 69403146 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 5 from 70465624 -[SCHEDULER ITER32 LR0.01679616]: preparing parameters end. -[SCHEDULER ITER32 LR0.01679616]: preparing layers... -(12:38:52 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:38:52 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:38:52 2015-11-18)[nerv] info: create layer: combinerL1 -(12:38:52 2015-11-18)[nerv] info: create layer: outputL -(12:38:52 2015-11-18)[nerv] info: create layer: softmaxL -(12:38:52 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER32 LR0.01679616]: preparing layers end. -[SCHEDULER ITER32 LR0.01679616]: Generate and initing TNN ... -<input> selectL1 0 -selectL1 recurrentL1 0 -recurrentL1 sigmoidL1 0 -sigmoidL1 combinerL1 0 -combinerL1 recurrentL1 1 -combinerL1 outputL 0 -outputL softmaxL 0 -<input> softmaxL 0 -softmaxL <output> 0 -recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 -outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 -sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 -softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER32 LR0.01679616]: Initing TNN end. -===ITERATION 32 LR 0.016796=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER32 LR0.01679616]: 40092 words processed Wed Nov 18 12:39:04 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.931442. - [global_conf.timer]: time spent on tnn_beforeprocess:0.49006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00626 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.71865 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.68077 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13372 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12770 clock time -[SCHEDULER ITER32 LR0.01679616]: 80099 words processed Wed Nov 18 12:39:16 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.928292. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48707 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98502 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.63892 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64141 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12398 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12751 clock time -[SCHEDULER ITER32 LR0.01679616]: 120004 words processed Wed Nov 18 12:39:28 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.923647. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48431 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.97694 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.63508 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64631 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12285 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12643 clock time -[SCHEDULER ITER32 LR0.01679616]: 160114 words processed Wed Nov 18 12:39:40 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.924997. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48575 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00642 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.67710 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65653 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12438 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12778 clock time -[SCHEDULER ITER32 LR0.01679616]: 200066 words processed Wed Nov 18 12:39:52 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.924254. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48340 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99875 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.67236 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65593 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12571 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13006 clock time -[SCHEDULER ITER32 LR0.01679616]: 240045 words processed Wed Nov 18 12:40:04 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.920565. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47752 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98278 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64194 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12063 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12662 clock time -[SCHEDULER ITER32 LR0.01679616]: 280057 words processed Wed Nov 18 12:40:16 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.916922. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47857 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99140 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.66105 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66069 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12368 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12821 clock time -[SCHEDULER ITER32 LR0.01679616]: 320106 words processed Wed Nov 18 12:40:28 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.915045. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.01313 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.71281 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13988 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12925 clock time -[SCHEDULER ITER32 LR0.01679616]: 360024 words processed Wed Nov 18 12:40:40 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.913357. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47887 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99731 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.66735 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65264 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12911 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12883 clock time -[SCHEDULER ITER32 LR0.01679616]: 400089 words processed Wed Nov 18 12:40:52 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.910512. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48028 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99099 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.68755 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13887 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12702 clock time -[SCHEDULER ITER32 LR0.01679616]: 440067 words processed Wed Nov 18 12:41:04 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.911129. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47680 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98253 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.63384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64296 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12935 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12726 clock time -[SCHEDULER ITER32 LR0.01679616]: 480051 words processed Wed Nov 18 12:41:16 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.911402. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00109 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.66080 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65426 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11984 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12747 clock time -[SCHEDULER ITER32 LR0.01679616]: 520140 words processed Wed Nov 18 12:41:28 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.910826. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47592 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98938 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65328 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65673 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12699 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12719 clock time -[SCHEDULER ITER32 LR0.01679616]: 560132 words processed Wed Nov 18 12:41:40 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.910163. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47474 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98268 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62839 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64559 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12444 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12717 clock time -[SCHEDULER ITER32 LR0.01679616]: 600118 words processed Wed Nov 18 12:41:52 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.907953. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47731 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99150 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.64494 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65301 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11857 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12746 clock time -[SCHEDULER ITER32 LR0.01679616]: 640090 words processed Wed Nov 18 12:42:04 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.905814. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48793 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.05643 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.81006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.69610 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14733 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13093 clock time -[SCHEDULER ITER32 LR0.01679616]: 680075 words processed Wed Nov 18 12:42:16 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.904486. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47539 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99296 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.64966 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12610 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12863 clock time -[SCHEDULER ITER32 LR0.01679616]: 720043 words processed Wed Nov 18 12:42:28 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.903383. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48215 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.02149 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.72594 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67253 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13533 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12929 clock time -[SCHEDULER ITER32 LR0.01679616]: 760012 words processed Wed Nov 18 12:42:40 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.901397. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48373 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.02926 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.74089 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67209 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14074 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13074 clock time -[SCHEDULER ITER32 LR0.01679616]: 800113 words processed Wed Nov 18 12:42:52 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.901182. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48676 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.04019 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.76458 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67916 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.14204 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13002 clock time -[SCHEDULER ITER32 LR0.01679616]: 840089 words processed Wed Nov 18 12:43:04 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.899902. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47685 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00183 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65355 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65120 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12682 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12583 clock time -[SCHEDULER ITER32 LR0.01679616]: 880052 words processed Wed Nov 18 12:43:16 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-1.899373. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46598 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92368 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46962 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.59446 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11070 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12162 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER32 LR0.01679616]: Displaying result: -[SCHEDULER ITER32 LR0.01679616]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 87.167707631587> <PPL_OOV 79.29226159407> <LOGP -1765504.0648689> -[SCHEDULER ITER32 LR0.01679616]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER32 LR0.01679616]: shuffling training file -===PEEK ON TEST 32=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER32 LR0.01679616]: 40087 words processed Wed Nov 18 12:43:25 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-2.106327. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47254 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93845 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.63866 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11638 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER32 LR0.01679616]: Displaying result: -[SCHEDULER ITER32 LR0.01679616]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 147.22052553121> <PPL_OOV 128.63441536524> <LOGP -173874.31210131> -[SCHEDULER ITER32 LR0.01679616]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. -===VALIDATION 32=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER32 LR0.01679616]: 40095 words processed Wed Nov 18 12:43:34 2015. - [SCHEDULER ITER32 LR0.01679616]: log prob per sample :-2.183727. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47546 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95362 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.65917 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11720 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER32 LR0.01679616]: Displaying result: -[SCHEDULER ITER32 LR0.01679616]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 151.5455837763> <PPL_OOV 135.67036887598> <LOGP -157292.09402864> -[SCHEDULER ITER32 LR0.01679616]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER32 LR0.01679616]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.32... - -[SCHEDULER ITER33 LR0.010077696]: preparing parameters... -[SCHEDULER ITER33 LR0.010077696]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.32... -reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 1 from 1062455 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 1066108 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 35822967 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 35939176 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 5 from 70465439 -[SCHEDULER ITER33 LR0.010077696]: preparing parameters end. -[SCHEDULER ITER33 LR0.010077696]: preparing layers... -(12:43:43 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:43:43 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:43:43 2015-11-18)[nerv] info: create layer: combinerL1 -(12:43:43 2015-11-18)[nerv] info: create layer: outputL -(12:43:43 2015-11-18)[nerv] info: create layer: softmaxL -(12:43:43 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER33 LR0.010077696]: preparing layers end. -[SCHEDULER ITER33 LR0.010077696]: Generate and initing TNN ... -<input> selectL1 0 -selectL1 recurrentL1 0 -recurrentL1 sigmoidL1 0 -sigmoidL1 combinerL1 0 -combinerL1 recurrentL1 1 -combinerL1 outputL 0 -outputL softmaxL 0 -<input> softmaxL 0 -softmaxL <output> 0 -recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 -outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 -sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 -softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER33 LR0.010077696]: Initing TNN end. -===ITERATION 33 LR 0.010078=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER33 LR0.010077696]: 40092 words processed Wed Nov 18 12:43:55 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.929093. - [global_conf.timer]: time spent on tnn_beforeprocess:0.49326 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.05184 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.80160 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.70523 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13696 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13018 clock time -[SCHEDULER ITER33 LR0.010077696]: 80099 words processed Wed Nov 18 12:44:07 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.925926. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47666 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93380 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.50971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61537 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10512 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12215 clock time -[SCHEDULER ITER33 LR0.010077696]: 120004 words processed Wed Nov 18 12:44:19 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.921375. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47511 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93011 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51037 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61482 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10676 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12276 clock time -[SCHEDULER ITER33 LR0.010077696]: 160114 words processed Wed Nov 18 12:44:31 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.922722. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47533 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93746 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.52086 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62463 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10575 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12221 clock time -[SCHEDULER ITER33 LR0.010077696]: 200066 words processed Wed Nov 18 12:44:43 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.921991. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95022 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.54922 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.63240 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10490 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12442 clock time -[SCHEDULER ITER33 LR0.010077696]: 240045 words processed Wed Nov 18 12:44:55 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.918333. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48672 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00091 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.63719 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64180 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11243 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12568 clock time -[SCHEDULER ITER33 LR0.010077696]: 280057 words processed Wed Nov 18 12:45:07 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.914713. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47773 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95508 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.57520 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64215 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11357 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12354 clock time -[SCHEDULER ITER33 LR0.010077696]: 320106 words processed Wed Nov 18 12:45:19 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.912862. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47212 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95449 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.55985 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.63461 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11271 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12378 clock time -[SCHEDULER ITER33 LR0.010077696]: 360024 words processed Wed Nov 18 12:45:31 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.911181. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46987 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94331 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51752 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.61670 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10461 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12202 clock time -[SCHEDULER ITER33 LR0.010077696]: 400089 words processed Wed Nov 18 12:45:43 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.908342. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47299 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95056 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.57284 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64702 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11576 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12435 clock time -[SCHEDULER ITER33 LR0.010077696]: 440067 words processed Wed Nov 18 12:45:55 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.908964. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47447 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.97302 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.59874 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65094 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11020 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12471 clock time -[SCHEDULER ITER33 LR0.010077696]: 480051 words processed Wed Nov 18 12:46:07 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.909244. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47210 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95866 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.56068 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.63818 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10737 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12379 clock time -[SCHEDULER ITER33 LR0.010077696]: 520140 words processed Wed Nov 18 12:46:19 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.908674. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46744 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.93725 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51020 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62134 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10777 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12119 clock time -[SCHEDULER ITER33 LR0.010077696]: 560132 words processed Wed Nov 18 12:46:31 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.908028. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46627 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92614 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.46808 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60445 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09855 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12077 clock time -[SCHEDULER ITER33 LR0.010077696]: 600118 words processed Wed Nov 18 12:46:43 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.905823. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46575 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.92047 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.45615 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09596 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12122 clock time -[SCHEDULER ITER33 LR0.010077696]: 640090 words processed Wed Nov 18 12:46:55 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.903686. - [global_conf.timer]: time spent on tnn_beforeprocess:0.46282 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.91642 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.44927 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.60102 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.09803 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12085 clock time -[SCHEDULER ITER33 LR0.010077696]: 680075 words processed Wed Nov 18 12:47:07 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.902356. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47094 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.94265 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.51933 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.62027 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10646 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12210 clock time -[SCHEDULER ITER33 LR0.010077696]: 720043 words processed Wed Nov 18 12:47:19 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.901253. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47270 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96782 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.58094 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64182 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.10872 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12466 clock time -[SCHEDULER ITER33 LR0.010077696]: 760012 words processed Wed Nov 18 12:47:32 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.899263. - [global_conf.timer]: time spent on tnn_beforeprocess:0.50005 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.16792 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:11.05536 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.77800 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.16164 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.13491 clock time -[SCHEDULER ITER33 LR0.010077696]: 800113 words processed Wed Nov 18 12:47:44 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.899044. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48191 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00530 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.68947 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.68089 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12088 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12640 clock time -[SCHEDULER ITER33 LR0.010077696]: 840089 words processed Wed Nov 18 12:47:56 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.897764. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47427 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00777 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.67260 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67305 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11776 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12626 clock time -[SCHEDULER ITER33 LR0.010077696]: 880052 words processed Wed Nov 18 12:48:08 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-1.897235. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47355 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.97415 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.59852 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64731 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11116 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12408 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER33 LR0.010077696]: Displaying result: -[SCHEDULER ITER33 LR0.010077696]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.729095282599> <PPL_OOV 78.902608908036> <LOGP -1763515.2633715> -[SCHEDULER ITER33 LR0.010077696]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER33 LR0.010077696]: shuffling training file -===PEEK ON TEST 33=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER33 LR0.010077696]: 40087 words processed Wed Nov 18 12:48:17 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-2.105025. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47679 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95951 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.66961 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11833 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER33 LR0.010077696]: Displaying result: -[SCHEDULER ITER33 LR0.010077696]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.687178305> <PPL_OOV 128.2464002936> <LOGP -173766.16450762> -[SCHEDULER ITER33 LR0.010077696]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. -===VALIDATION 33=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER33 LR0.010077696]: 40095 words processed Wed Nov 18 12:48:27 2015. - [SCHEDULER ITER33 LR0.010077696]: log prob per sample :-2.182599. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48786 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99972 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.72529 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11994 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER33 LR0.010077696]: Displaying result: -[SCHEDULER ITER33 LR0.010077696]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 151.00228818473> <PPL_OOV 135.25645893787> <LOGP -157194.21505764> -[SCHEDULER ITER33 LR0.010077696]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER33 LR0.010077696]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.33... - -[SCHEDULER ITER34 LR0.0060466176]: preparing parameters... -[SCHEDULER ITER34 LR0.0060466176]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.33... -reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 1 from 34526300 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 34529953 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 69286738 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 69402945 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 5 from 70465377 -[SCHEDULER ITER34 LR0.0060466176]: preparing parameters end. -[SCHEDULER ITER34 LR0.0060466176]: preparing layers... -(12:48:36 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:48:36 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:48:36 2015-11-18)[nerv] info: create layer: combinerL1 -(12:48:36 2015-11-18)[nerv] info: create layer: outputL -(12:48:36 2015-11-18)[nerv] info: create layer: softmaxL -(12:48:36 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER34 LR0.0060466176]: preparing layers end. -[SCHEDULER ITER34 LR0.0060466176]: Generate and initing TNN ... -<input> selectL1 0 -selectL1 recurrentL1 0 -recurrentL1 sigmoidL1 0 -sigmoidL1 combinerL1 0 -combinerL1 recurrentL1 1 -combinerL1 outputL 0 -outputL softmaxL 0 -<input> softmaxL 0 -softmaxL <output> 0 -recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 -outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 -sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 -softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER34 LR0.0060466176]: Initing TNN end. -===ITERATION 34 LR 0.006047=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER34 LR0.0060466176]: 40092 words processed Wed Nov 18 12:48:48 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.927516. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00205 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.68031 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67014 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13007 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12469 clock time -[SCHEDULER ITER34 LR0.0060466176]: 80099 words processed Wed Nov 18 12:49:00 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.924363. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48701 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00152 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.63440 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64010 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11586 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12344 clock time -[SCHEDULER ITER34 LR0.0060466176]: 120004 words processed Wed Nov 18 12:49:12 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.919896. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48565 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00140 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.64750 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64753 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11754 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12436 clock time -[SCHEDULER ITER34 LR0.0060466176]: 160114 words processed Wed Nov 18 12:49:24 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.921244. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48556 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.01856 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.69348 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66448 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13001 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12436 clock time -[SCHEDULER ITER34 LR0.0060466176]: 200066 words processed Wed Nov 18 12:49:36 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.920525. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48122 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98935 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62497 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64569 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11758 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12284 clock time -[SCHEDULER ITER34 LR0.0060466176]: 240045 words processed Wed Nov 18 12:49:48 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.916893. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47968 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99230 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62215 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64115 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11871 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12411 clock time -[SCHEDULER ITER34 LR0.0060466176]: 280057 words processed Wed Nov 18 12:50:00 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.913290. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48030 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98573 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62934 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64805 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12454 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12336 clock time -[SCHEDULER ITER34 LR0.0060466176]: 320106 words processed Wed Nov 18 12:50:12 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.911461. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48191 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00146 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.66001 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66010 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12166 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12293 clock time -[SCHEDULER ITER34 LR0.0060466176]: 360024 words processed Wed Nov 18 12:50:24 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.909783. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47684 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99920 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65297 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65507 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12505 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12423 clock time -[SCHEDULER ITER34 LR0.0060466176]: 400089 words processed Wed Nov 18 12:50:36 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.906950. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47317 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98452 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62992 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65181 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12892 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12372 clock time -[SCHEDULER ITER34 LR0.0060466176]: 440067 words processed Wed Nov 18 12:50:48 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.907576. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47456 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98303 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.60790 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64580 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11456 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12380 clock time -[SCHEDULER ITER34 LR0.0060466176]: 480051 words processed Wed Nov 18 12:51:00 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.907859. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47196 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98019 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.58384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.63203 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11347 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12376 clock time -[SCHEDULER ITER34 LR0.0060466176]: 520140 words processed Wed Nov 18 12:51:12 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.907297. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48169 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.02324 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.71107 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67375 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13270 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12569 clock time -[SCHEDULER ITER34 LR0.0060466176]: 560132 words processed Wed Nov 18 12:51:24 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.906662. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47579 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.01046 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65603 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65982 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11530 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12477 clock time -[SCHEDULER ITER34 LR0.0060466176]: 600118 words processed Wed Nov 18 12:51:36 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.904460. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47596 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99203 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62056 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64379 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11688 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12410 clock time -[SCHEDULER ITER34 LR0.0060466176]: 640090 words processed Wed Nov 18 12:51:48 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.902324. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47762 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00946 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.66148 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65471 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12357 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12500 clock time -[SCHEDULER ITER34 LR0.0060466176]: 680075 words processed Wed Nov 18 12:52:00 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.900997. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48074 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.01651 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.67812 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65846 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12636 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12525 clock time -[SCHEDULER ITER34 LR0.0060466176]: 720043 words processed Wed Nov 18 12:52:12 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.899895. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48134 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00281 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65324 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65169 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12287 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12469 clock time -[SCHEDULER ITER34 LR0.0060466176]: 760012 words processed Wed Nov 18 12:52:24 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.897902. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47873 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.03051 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.70760 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66441 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13356 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12620 clock time -[SCHEDULER ITER34 LR0.0060466176]: 800113 words processed Wed Nov 18 12:52:36 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.897682. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.03194 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.71910 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67635 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13007 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12576 clock time -[SCHEDULER ITER34 LR0.0060466176]: 840089 words processed Wed Nov 18 12:52:48 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.896402. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47695 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00887 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65231 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64851 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12427 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12525 clock time -[SCHEDULER ITER34 LR0.0060466176]: 880052 words processed Wed Nov 18 12:53:00 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-1.895871. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48090 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00792 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65612 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64769 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12460 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12384 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER34 LR0.0060466176]: Displaying result: -[SCHEDULER ITER34 LR0.0060466176]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.450253074165> <PPL_OOV 78.65525540066> <LOGP -1762247.6592947> -[SCHEDULER ITER34 LR0.0060466176]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER34 LR0.0060466176]: shuffling training file -===PEEK ON TEST 34=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER34 LR0.0060466176]: 40087 words processed Wed Nov 18 12:53:09 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-2.104154. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48671 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.02959 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.75717 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12063 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER34 LR0.0060466176]: Displaying result: -[SCHEDULER ITER34 LR0.0060466176]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.35597848968> <PPL_OOV 127.97734439262> <LOGP -173690.9809428> -[SCHEDULER ITER34 LR0.0060466176]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. -===VALIDATION 34=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER34 LR0.0060466176]: 40095 words processed Wed Nov 18 12:53:19 2015. - [SCHEDULER ITER34 LR0.0060466176]: log prob per sample :-2.181872. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48545 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.01889 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.74313 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12000 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER34 LR0.0060466176]: Displaying result: -[SCHEDULER ITER34 LR0.0060466176]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 150.68103244818> <PPL_OOV 134.98934891245> <LOGP -157130.89131439> -[SCHEDULER ITER34 LR0.0060466176]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER34 LR0.0060466176]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.34... - -[SCHEDULER ITER35 LR0.00362797056]: preparing parameters... -[SCHEDULER ITER35 LR0.00362797056]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.34... -reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} - -reading chunk 1 from 1062432 -metadata: return {type="nerv.BiasParam",id="bp_h"} - -reading chunk 2 from 1066085 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} - -reading chunk 3 from 35822859 -metadata: return {type="nerv.BiasParam",id="bp_o"} - -reading chunk 4 from 35939065 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} - -reading chunk 5 from 70465345 -[SCHEDULER ITER35 LR0.00362797056]: preparing parameters end. -[SCHEDULER ITER35 LR0.00362797056]: preparing layers... -(12:53:29 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:53:29 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:53:29 2015-11-18)[nerv] info: create layer: combinerL1 -(12:53:29 2015-11-18)[nerv] info: create layer: outputL -(12:53:29 2015-11-18)[nerv] info: create layer: softmaxL -(12:53:29 2015-11-18)[nerv] info: create layer: selectL1 -[SCHEDULER ITER35 LR0.00362797056]: preparing layers end. -[SCHEDULER ITER35 LR0.00362797056]: Generate and initing TNN ... -<input> selectL1 0 -selectL1 recurrentL1 0 -recurrentL1 sigmoidL1 0 -sigmoidL1 combinerL1 0 -combinerL1 recurrentL1 1 -combinerL1 outputL 0 -outputL softmaxL 0 -<input> softmaxL 0 -softmaxL <output> 0 -recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 -outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 -sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 -selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 -softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL -[SCHEDULER ITER35 LR0.00362797056]: Initing TNN end. -===ITERATION 35 LR 0.003628=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER35 LR0.00362797056]: 40092 words processed Wed Nov 18 12:53:41 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.926501. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48931 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.03093 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.74216 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.68919 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13462 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12596 clock time -[SCHEDULER ITER35 LR0.00362797056]: 80099 words processed Wed Nov 18 12:53:53 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.923356. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48969 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99884 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65501 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65643 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11986 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12518 clock time -[SCHEDULER ITER35 LR0.00362797056]: 120004 words processed Wed Nov 18 12:54:05 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.918930. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99255 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.64961 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65397 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12042 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12487 clock time -[SCHEDULER ITER35 LR0.00362797056]: 160114 words processed Wed Nov 18 12:54:17 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.920291. - [global_conf.timer]: time spent on tnn_beforeprocess:0.49137 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00943 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.70487 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12998 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12556 clock time -[SCHEDULER ITER35 LR0.00362797056]: 200066 words processed Wed Nov 18 12:54:29 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.919579. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48742 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00354 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.67469 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66238 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12639 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12564 clock time -[SCHEDULER ITER35 LR0.00362797056]: 240045 words processed Wed Nov 18 12:54:41 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.915968. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48365 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00870 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.67120 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66389 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11996 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12644 clock time -[SCHEDULER ITER35 LR0.00362797056]: 280057 words processed Wed Nov 18 12:54:53 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.912378. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48882 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.02058 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.72367 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.68383 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13038 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12643 clock time -[SCHEDULER ITER35 LR0.00362797056]: 320106 words processed Wed Nov 18 12:55:05 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.910563. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48681 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.02039 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.72114 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.68362 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13112 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12607 clock time -[SCHEDULER ITER35 LR0.00362797056]: 360024 words processed Wed Nov 18 12:55:17 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.908889. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48102 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00263 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.66823 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66255 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12503 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12609 clock time -[SCHEDULER ITER35 LR0.00362797056]: 400089 words processed Wed Nov 18 12:55:29 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.906061. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48152 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00318 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.69459 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67890 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.13412 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12647 clock time -[SCHEDULER ITER35 LR0.00362797056]: 440067 words processed Wed Nov 18 12:55:41 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.906689. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00280 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.65645 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65982 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11976 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12703 clock time -[SCHEDULER ITER35 LR0.00362797056]: 480051 words processed Wed Nov 18 12:55:53 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.906977. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48161 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00954 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.66486 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.66291 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11722 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12708 clock time -[SCHEDULER ITER35 LR0.00362797056]: 520140 words processed Wed Nov 18 12:56:05 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.906417. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47997 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.01201 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.69293 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67482 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12926 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12683 clock time -[SCHEDULER ITER35 LR0.00362797056]: 560132 words processed Wed Nov 18 12:56:17 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.905791. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48136 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.99635 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.64164 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65589 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11674 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12566 clock time -[SCHEDULER ITER35 LR0.00362797056]: 600118 words processed Wed Nov 18 12:56:29 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.903593. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47623 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98530 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62253 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64999 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12104 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12545 clock time -[SCHEDULER ITER35 LR0.00362797056]: 640090 words processed Wed Nov 18 12:56:41 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.901457. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47975 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98858 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.63113 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64914 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12413 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12474 clock time -[SCHEDULER ITER35 LR0.00362797056]: 680075 words processed Wed Nov 18 12:56:53 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.900130. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47742 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98445 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.61271 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64424 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11973 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12440 clock time -[SCHEDULER ITER35 LR0.00362797056]: 720043 words processed Wed Nov 18 12:57:05 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.899031. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48383 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.01215 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.68746 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.67444 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11971 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12678 clock time -[SCHEDULER ITER35 LR0.00362797056]: 760012 words processed Wed Nov 18 12:57:17 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.897035. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47436 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98906 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.62650 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65730 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11432 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12553 clock time -[SCHEDULER ITER35 LR0.00362797056]: 800113 words processed Wed Nov 18 12:57:29 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.896814. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47329 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.97448 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.60850 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.65262 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.12034 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12447 clock time -[SCHEDULER ITER35 LR0.00362797056]: 840089 words processed Wed Nov 18 12:57:41 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.895535. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47191 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.95956 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.55920 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.63377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11201 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12305 clock time -[SCHEDULER ITER35 LR0.00362797056]: 880052 words processed Wed Nov 18 12:57:53 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-1.895002. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47568 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.96820 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:10.58743 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.64028 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.11656 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.12360 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER35 LR0.00362797056]: Displaying result: -[SCHEDULER ITER35 LR0.00362797056]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.272834134732> <PPL_OOV 78.498152777863> <LOGP -1761440.4892072> -[SCHEDULER ITER35 LR0.00362797056]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. -[SCHEDULER ITER35 LR0.00362797056]: shuffling training file -===PEEK ON TEST 35=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER35 LR0.00362797056]: 40087 words processed Wed Nov 18 12:58:02 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-2.103584. - [global_conf.timer]: time spent on tnn_beforeprocess:0.47962 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:2.98519 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.69977 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11922 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER35 LR0.00362797056]: Displaying result: -[SCHEDULER ITER35 LR0.00362797056]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.17039447141> <PPL_OOV 127.79140045143> <LOGP -173638.92932372> -[SCHEDULER ITER35 LR0.00362797056]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. -===VALIDATION 35=== -[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER ITER35 LR0.00362797056]: 40095 words processed Wed Nov 18 12:58:12 2015. - [SCHEDULER ITER35 LR0.00362797056]: log prob per sample :-2.181381. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48480 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00216 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.72472 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11993 clock time -[LOG]LMSeqReader: file expires, closing. -[SCHEDULER ITER35 LR0.00362797056]: Displaying result: -[SCHEDULER ITER35 LR0.00362797056]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 150.51338825824> <PPL_OOV 134.81957448081> <LOGP -157090.57774537> -[SCHEDULER ITER35 LR0.00362797056]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER35 LR0.00362797056]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.35... - +(18:42:58 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false +[SCHEDULER ITER30 LR0.0060466176]: Displaying result: +[SCHEDULER ITER30 LR0.0060466176]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 150.62054355245> <PPL_OOV 134.98729348231> <LOGP -157130.40354815> +[SCHEDULER ITER30 LR0.0060466176]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER30 LR0.0060466176]: PPL improves, saving net to file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.30... + +(18:43:03 2015-12-23)[nerv] info: saving final nn to param.final ===VALIDATION PPL record=== -<ITER0 LR0.00000 train:0.000 valid:10092.059 test:0.000> -<ITER1 LR1.00000 train:447.200 valid:331.713 test:323.544> -<ITER2 LR1.00000 train:290.374 valid:252.806 test:242.776> -<ITER3 LR1.00000 train:245.397 valid:228.358 test:218.981> -<ITER4 LR1.00000 train:212.502 valid:214.052 test:205.791> -<ITER5 LR1.00000 train:195.668 valid:203.704 test:193.918> -<ITER6 LR1.00000 train:180.480 valid:196.170 test:187.550> -<ITER7 LR1.00000 train:170.451 valid:187.462 test:178.690> -<ITER8 LR1.00000 train:162.949 valid:183.632 test:175.370> -<ITER9 LR1.00000 train:156.371 valid:179.375 test:171.281> -<ITER10 LR1.00000 train:150.464 valid:177.420 test:168.900> -<ITER11 LR1.00000 train:145.764 valid:174.335 test:165.486> -<ITER12 LR1.00000 train:141.373 valid:172.803 test:163.822> -<ITER13 LR1.00000 train:137.631 valid:172.220 test:163.131> -<ITER14 LR1.00000 train:134.414 valid:169.850 test:160.997> -<ITER15 LR1.00000 train:131.360 valid:168.661 test:160.001> -<ITER16 LR1.00000 train:128.344 valid:166.812 test:158.475> -<ITER17 LR1.00000 train:127.214 valid:166.572 test:158.413> -<ITER18 LR1.00000 train:123.861 valid:165.108 test:156.954> -<ITER19 LR1.00000 train:121.666 valid:165.028 test:156.988> -<ITER20 LR1.00000 train:119.543 valid:164.221 test:156.194> -<ITER21 LR1.00000 train:117.486 valid:163.877 test:155.830> -<ITER22 LR1.00000 train:115.576 valid:163.700 test:155.498> -<ITER23 LR1.00000 train:114.595 valid:162.328 test:154.452> -<ITER24 LR1.00000 train:112.775 valid:163.327 test:155.128> -<ITER25 LR0.60000 train:100.177 valid:152.806 test:145.172> -<ITER26 LR0.36000 train:92.223 valid:146.958 test:139.509> -<ITER27 LR0.21600 train:87.331 valid:142.745 test:135.398> -<ITER28 LR0.12960 train:84.175 valid:140.135 test:132.836> -<ITER29 LR0.07776 train:82.103 valid:138.447 test:131.203> -<ITER30 LR0.04666 train:80.784 valid:137.161 test:130.007> -<ITER31 LR0.02799 train:79.906 valid:136.301 test:129.205> -<ITER32 LR0.01680 train:79.292 valid:135.670 test:128.634> -<ITER33 LR0.01008 train:78.903 valid:135.256 test:128.246> -<ITER34 LR0.00605 train:78.655 valid:134.989 test:127.977> -<ITER35 LR0.00363 train:78.498 valid:134.820 test:127.791> +<ITER0 LR0.00000 train:0.000 valid:9812.822 test:0.000> +<ITER1 LR1.00000 train:421.501 valid:324.020 test:313.387> +<ITER2 LR1.00000 train:289.556 valid:255.815 test:244.839> +<ITER3 LR1.00000 train:236.754 valid:226.305 test:216.081> +<ITER4 LR1.00000 train:226.168 valid:214.109 test:203.641> +<ITER5 LR1.00000 train:195.663 valid:202.457 test:192.304> +<ITER6 LR1.00000 train:182.226 valid:192.223 test:182.833> +<ITER7 LR1.00000 train:170.095 valid:185.946 test:176.566> +<ITER8 LR1.00000 train:161.408 valid:181.148 test:171.973> +<ITER9 LR1.00000 train:153.718 valid:176.130 test:167.211> +<ITER10 LR1.00000 train:147.780 valid:173.685 test:164.491> +<ITER11 LR1.00000 train:149.926 valid:172.805 test:163.484> +<ITER12 LR1.00000 train:140.757 valid:170.455 test:161.232> +<ITER13 LR1.00000 train:135.085 valid:168.364 test:159.029> +<ITER14 LR1.00000 train:131.778 valid:166.388 test:157.108> +<ITER15 LR1.00000 train:128.261 valid:165.031 test:155.693> +<ITER16 LR1.00000 train:125.367 valid:163.737 test:154.625> +<ITER17 LR1.00000 train:122.259 valid:163.165 test:153.833> +<ITER18 LR1.00000 train:119.746 valid:162.024 test:152.709> +<ITER19 LR1.00000 train:117.434 valid:161.057 test:152.106> +<ITER20 LR1.00000 train:115.305 valid:161.022 test:152.145> +<ITER21 LR0.60000 train:101.289 valid:152.727 test:144.471> +<ITER22 LR0.36000 train:93.239 valid:145.621 test:137.835> +<ITER23 LR0.21600 train:88.190 valid:141.804 test:134.255> +<ITER24 LR0.12960 train:84.884 valid:139.284 test:131.895> +<ITER25 LR0.07776 train:82.704 valid:137.681 test:130.387> +<ITER26 LR0.04666 train:81.310 valid:136.558 test:129.294> +<ITER27 LR0.02799 train:80.379 valid:135.966 test:128.732> +<ITER28 LR0.01680 train:79.734 valid:135.527 test:128.374> +<ITER29 LR0.01008 train:79.327 valid:135.207 test:128.130> +<ITER30 LR0.00605 train:79.069 valid:134.987 test:127.969> ===FINAL TEST=== [SCHEDULER FINAL_TEST]: preparing parameters... -[SCHEDULER FINAL_TEST]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.35... +[SCHEDULER FINAL_TEST]: loading parameter from file /home/slhome/txh18/workspace/ptb/EXP-nerv/rnnlm_tnnh300ch15ba10slr1wc1e-06/params.30... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.BiasParam",id="recurrentL1_bp"} -reading chunk 1 from 34526247 -metadata: return {type="nerv.BiasParam",id="bp_h"} +reading chunk 1 from 3663 +metadata: return {type="nerv.BiasParam",id="outputL_bp"} -reading chunk 2 from 34529900 -metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} +reading chunk 2 from 119934 +metadata: return {type="nerv.LinearTransParam",id="recurrentL1_ltp_hh"} -reading chunk 3 from 69286627 -metadata: return {type="nerv.BiasParam",id="bp_o"} +reading chunk 3 from 1182997 +metadata: return {type="nerv.LinearTransParam",id="outputL_ltp"} -reading chunk 4 from 69402830 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 35917495 +metadata: return {type="nerv.LinearTransParam",id="selectL1_ltp"} -reading chunk 5 from 70465248 +reading chunk 5 from 70454370 [SCHEDULER FINAL_TEST]: preparing parameters end. [SCHEDULER FINAL_TEST]: preparing layers... -(12:58:22 2015-11-18)[nerv] info: create layer: recurrentL1 -(12:58:22 2015-11-18)[nerv] info: create layer: sigmoidL1 -(12:58:22 2015-11-18)[nerv] info: create layer: combinerL1 -(12:58:22 2015-11-18)[nerv] info: create layer: outputL -(12:58:22 2015-11-18)[nerv] info: create layer: softmaxL -(12:58:22 2015-11-18)[nerv] info: create layer: selectL1 +(18:43:04 2015-12-23)[nerv] info: create layer: sigmoidL1 +(18:43:04 2015-12-23)[nerv] info: create layer: combinerL1 +(18:43:04 2015-12-23)[nerv] info: create layer: selectL1 +(18:43:04 2015-12-23)[nerv] info: Param [ltp] of layer [selectL1] found in layer_conf.paramRepo. +(18:43:04 2015-12-23)[nerv] info: create layer: outputL +(18:43:04 2015-12-23)[nerv] info: Param [ltp] of layer [outputL] found in layer_conf.paramRepo. +(18:43:04 2015-12-23)[nerv] info: Param [bp] of layer [outputL] found in layer_conf.paramRepo. +(18:43:04 2015-12-23)[nerv] info: create layer: softmaxL +(18:43:04 2015-12-23)[nerv] info: create layer: recurrentL1 +(18:43:04 2015-12-23)[nerv] info: Param [bp] of layer [recurrentL1] found in layer_conf.paramRepo. +(18:43:04 2015-12-23)[nerv] info: Param [ltp_hh] of layer [recurrentL1] found in layer_conf.paramRepo. [SCHEDULER FINAL_TEST]: preparing layers end. [SCHEDULER FINAL_TEST]: Generate and initing TNN ... +(18:43:04 2015-12-23)[nerv] info: tnn(TNN) will extend storage beyond MB border for time steps 5... <input> selectL1 0 selectL1 recurrentL1 0 recurrentL1 sigmoidL1 0 @@ -9351,22 +9128,32 @@ combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 -TNN initing storage selectL1 -> recurrentL1 -TNN initing storage recurrentL1 -> sigmoidL1 -TNN initing storage sigmoidL1 -> combinerL1 -TNN initing storage combinerL1 -> recurrentL1 -TNN initing storage combinerL1 -> outputL -TNN initing storage outputL -> softmaxL +(18:43:04 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:43:04 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:43:04 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:43:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:43:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:43:04 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL [SCHEDULER FINAL_TEST]: Initing TNN end. [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... -batch_size: 10 chunk_size 15 -[SCHEDULER FINAL_TEST]: 40087 words processed Wed Nov 18 12:58:26 2015. - [SCHEDULER FINAL_TEST]: log prob per sample :-2.103584. - [global_conf.timer]: time spent on tnn_beforeprocess:0.48147 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.00189 clock time - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:3.71964 clock time - [global_conf.timer]: time spent on tnn_afterprocess:0.11812 clock time +[LOG]LMSeqReader: batch_size: 10 chunk_size 15 +[LOG]LMSeqReader: se_mode: false +(18:43:04 2015-12-23)[nerv] info: TNN initing storage selectL1->recurrentL1 +(18:43:04 2015-12-23)[nerv] info: TNN initing storage recurrentL1->sigmoidL1 +(18:43:04 2015-12-23)[nerv] info: TNN initing storage sigmoidL1->combinerL1 +(18:43:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->recurrentL1 +(18:43:04 2015-12-23)[nerv] info: TNN initing storage combinerL1->outputL +(18:43:04 2015-12-23)[nerv] info: TNN initing storage outputL->softmaxL +(18:43:04 2015-12-23)[nerv] info: LMTrainer.lm_process_file_rnn: begin processing... +[SCHEDULER FINAL_TEST]: 40087 words processed Wed Dec 23 18:43:12 2015. + [SCHEDULER FINAL_TEST]: log prob per sample :-2.106035. + [global_conf.timer]: time spent on tnn_beforeprocess:0.49203 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:6.12557 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:6.88028 clock time + [global_conf.timer]: time spent on tnn_afterprocess:0.14512 clock time [LOG]LMSeqReader: file expires, closing. +(18:43:20 2015-12-23)[nerv] info: lmseqreader file ends, printing stats... +al_sen_start:false [SCHEDULER FINAL_TEST]: Displaying result: -[SCHEDULER FINAL_TEST]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.17039441757> <PPL_OOV 127.79140027029> <LOGP -173638.92927297> +[SCHEDULER FINAL_TEST]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.23750831203> <PPL_OOV 127.96908067181> <LOGP -173688.669271> [SCHEDULER FINAL_TEST]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index 6e3fab9..9bdd5ff 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -333,7 +333,7 @@ end lr_half = false --can not be local, to be set by loadstring start_iter = -1 -start_lr = global_conf.lrate +start_lr = nil ppl_last = 100000 commands_str = "train:test" commands = {} @@ -358,7 +358,9 @@ global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str .. global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') commands = nerv.SUtil.parse_commands_set(commands_str) -global_conf.lrate = start_lr +if start_lr ~= nil then + global_conf.lrate = start_lr +end nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) nerv.LMUtil.wait(2) diff --git a/nerv/examples/lmptb/rnnlm_ptb_main.lua b/nerv/examples/lmptb/rnnlm_ptb_main.lua index ca62023..dc011fb 100644 --- a/nerv/examples/lmptb/rnnlm_ptb_main.lua +++ b/nerv/examples/lmptb/rnnlm_ptb_main.lua @@ -77,7 +77,7 @@ function prepare_layers(global_conf) local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} local layers = { - ["nerv.AffineRecurrentLayer"] = { + ["nerv.AffineRecurrentPlusVecLayer"] = { ["recurrentL1"] = recurrentLconfig, }, @@ -163,9 +163,11 @@ local train_fn, valid_fn, test_fn global_conf = {} local set = arg[1] --"test" +root_dir = '/home/slhome/txh18/workspace' + if (set == "ptb") then -data_dir = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata' +data_dir = root_dir .. '/nerv/nerv/nerv/examples/lmptb/PTBdata' train_fn = data_dir .. '/ptb.train.txt.adds' valid_fn = data_dir .. '/ptb.valid.txt.adds' test_fn = data_dir .. '/ptb.test.txt.adds' @@ -177,10 +179,10 @@ global_conf = { mmat_type = nerv.MMatrixFloat, nn_act_default = 0, - hidden_size = 400, --set to 400 for a stable good test PPL + hidden_size = 300, --set to 400 for a stable good test PPL chunk_size = 15, batch_size = 10, - max_iter = 35, + max_iter = 30, decay_iter = 15, param_random = function() return (math.random() / 5 - 0.1) end, @@ -188,10 +190,11 @@ global_conf = { valid_fn = valid_fn, test_fn = test_fn, vocab_fn = vocab_fn, + max_sen_len = 90, sche_log_pre = "[SCHEDULER]:", log_w_num = 40000, --give a message when log_w_num words have been processed timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/nerv/play/ptbEXP/tnn_test' + work_dir_base = root_dir .. '/ptb/EXP-nerv/rnnlm_tnn' } elseif (set == "msr_sc") then @@ -259,6 +262,8 @@ end lr_half = false --can not be local, to be set by loadstring start_iter = -1 ppl_last = 100000 +test_iter = -1 +commands_str = "train:test" if (arg[2] ~= nil) then printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) loadstring(arg[2])() @@ -271,6 +276,9 @@ global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_si global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' global_conf.param_fn = global_conf.work_dir .. "/params" +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str .. os.date("_TT%m_%d_%X",os.time()) +global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') +commands = nerv.SUtil.parse_commands_set(commands_str) ----------------printing options--------------------------------- printf("%s printing global_conf...\n", global_conf.sche_log_pre) @@ -281,92 +289,127 @@ nerv.LMUtil.wait(2) printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) print("lr_half", lr_half) print("start_iter", start_iter) +print("test_iter", test_iter) print("ppl_last", ppl_last) printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) nerv.LMUtil.wait(2) ------------------printing options end------------------------------ -math.randomseed(1) - printf("%s creating work_dir...\n", global_conf.sche_log_pre) -os.execute("mkdir -p "..global_conf.work_dir) +os.execute("mkdir -p ".. global_conf.work_dir) os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf) +--redirecting log outputs! +nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) + +math.randomseed(1) + local vocab = nerv.LMVocab() global_conf["vocab"] = vocab printf("%s building vocab...\n", global_conf.sche_log_pre) global_conf.vocab:build_file(global_conf.vocab_fn, false) ppl_rec = {} -if start_iter == -1 then - prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file -end +local final_iter = -1 +if commands["train"] == 1 then + if start_iter == -1 then + prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file + end + + if start_iter == -1 or start_iter == 0 then + print("===INITIAL VALIDATION===") + local tnn = load_net(global_conf, 0) + global_conf.paramRepo = tnn:get_params() --get auto-generted params + global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file + local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + nerv.LMUtil.wait(1) + ppl_rec[0] = {} + ppl_rec[0].valid = result:ppl_all("rnn") + ppl_last = ppl_rec[0].valid + ppl_rec[0].train = 0 + ppl_rec[0].test = 0 + ppl_rec[0].lr = 0 + + start_iter = 1 + + print() + end + + for iter = start_iter, global_conf.max_iter, 1 do + final_iter = iter --for final testing + global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" + tnn = load_net(global_conf, iter - 1) + printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! + ppl_rec[iter] = {} + ppl_rec[iter].train = result:ppl_all("rnn") + --shuffling training file + printf("%s shuffling training file\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) + os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) + printf("===PEEK ON TEST %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! + ppl_rec[iter].test = result:ppl_all("rnn") + printf("===VALIDATION %d===\n", iter) + result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! + ppl_rec[iter].valid = result:ppl_all("rnn") + ppl_rec[iter].lr = global_conf.lrate + if ((ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true) and iter > global_conf.decay_iter) then + global_conf.lrate = (global_conf.lrate * 0.6) + end + if ppl_rec[iter].valid < ppl_last then + printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) + global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) + else + printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) + os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) + end + if ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true then + lr_half = true + end + if ppl_rec[iter].valid < ppl_last then + ppl_last = ppl_rec[iter].valid + end + printf("\n") + nerv.LMUtil.wait(2) + end -if start_iter == -1 or start_iter == 0 then - print("===INITIAL VALIDATION===") - local tnn = load_net(global_conf, 0) - global_conf.paramRepo = tnn:get_params() --get auto-generted params - global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file - local result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! - nerv.LMUtil.wait(1) - ppl_rec[0] = {} - ppl_rec[0].valid = result:ppl_all("rnn") - ppl_last = ppl_rec[0].valid - ppl_rec[0].train = 0 - ppl_rec[0].test = 0 - ppl_rec[0].lr = 0 - - start_iter = 1 - - print() -end + nerv.info("saving final nn to param.final") + os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final') -local final_iter -for iter = start_iter, global_conf.max_iter, 1 do - final_iter = iter --for final testing - global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" - tnn = load_net(global_conf, iter - 1) - printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update! - ppl_rec[iter] = {} - ppl_rec[iter].train = result:ppl_all("rnn") - --shuffling training file - printf("%s shuffling training file\n", global_conf.sche_log_pre) - os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak) - os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf) - printf("===PEEK ON TEST %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! - ppl_rec[iter].test = result:ppl_all("rnn") - printf("===VALIDATION %d===\n", iter) - result = LMTrainer.lm_process_file_rnn(global_conf, global_conf.valid_fn, tnn, false) --false update! - ppl_rec[iter].valid = result:ppl_all("rnn") - ppl_rec[iter].lr = global_conf.lrate - if ((ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true) and iter > global_conf.decay_iter) then - global_conf.lrate = (global_conf.lrate * 0.6) - end - if ppl_rec[iter].valid < ppl_last then - printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) - global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) - else - printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) - os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter)) + printf("===VALIDATION PPL record===\n") + for i, _ in pairs(ppl_rec) do + printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) end - if ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true then - lr_half = true + printf("\n") +end --if commands["train"] + +if commands["test"] == 1 then + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter end - if ppl_rec[iter].valid < ppl_last then - ppl_last = ppl_rec[iter].valid + if test_iter == -1 then + test_iter = "final" end - printf("\n") - nerv.LMUtil.wait(2) -end -printf("===VALIDATION PPL record===\n") -for i, _ in pairs(ppl_rec) do - printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) -end -printf("\n") -printf("===FINAL TEST===\n") -global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" -tnn = load_net(global_conf, final_iter) -LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! + + printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + tnn = load_net(global_conf, test_iter) + LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["test"] + +if commands["wordprob"] == 1 then + if final_iter ~= -1 and test_iter == -1 then + test_iter = final_iter + end + if test_iter == -1 then + test_iter = "final" + end + + printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + tnn = load_net(global_conf, test_iter) + LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false, {["word_prob_report"] = true}) --false update! +end --if commands["test"] diff --git a/nerv/examples/lmptb/sample_grulm_ptb_main.lua b/nerv/examples/lmptb/sample_grulm_ptb_main.lua new file mode 100644 index 0000000..9a13d36 --- /dev/null +++ b/nerv/examples/lmptb/sample_grulm_ptb_main.lua @@ -0,0 +1,440 @@ +require 'lmptb.lmvocab' +require 'lmptb.lmfeeder' +require 'lmptb.lmutil' +require 'lmptb.layer.init' +--require 'tnn.init' +require 'lmptb.lmseqreader' +require 'lm_trainer' +require 'lm_sampler' + +--[[global function rename]]-- +--local printf = nerv.printf +local LMTrainer = nerv.LMTrainer +--[[global function rename ends]]-- + +function prepare_parameters(global_conf, fn) + nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) + + global_conf.paramRepo = nerv.ParamRepo() + local paramRepo = global_conf.paramRepo + + nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, fn) + paramRepo:import({fn}, nil, global_conf) + + nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) + + return nil +end + +--global_conf: table +--Returns: nerv.LayerRepo +function prepare_layers(global_conf) + nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) + + local pr = global_conf.paramRepo + + local du = false + + --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}} + --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} + + local layers = { + ["nerv.GRULayerT"] = { + ["gruL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}}, + }, + + ["nerv.DropoutLayerT"] = { + ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SelectLinearLayer"] = { + ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, + }, + + ["nerv.CombinerLayer"] = { + ["combinerL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + }, + + ["nerv.AffineLayer"] = { + ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, + }, + + ["nerv.SoftmaxCELayerT"] = { + ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, + }, + } + + for l = 2, global_conf.layer_num do + layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.GRULayerT"]["gruL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}} + layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} + end + --[[ --we do not need those in the new tnn framework + printf("%s adding %d bptt layers...\n", global_conf.sche_log_pre, global_conf.bptt) + for i = 1, global_conf.bptt do + layers["nerv.IndRecurrentLayer"]["recurrentL" .. (i + 1)] = recurrentLconfig + layers["nerv.SigmoidLayer"]["sigmoidL" .. (i + 1)] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.SelectLinearLayer"]["selectL" .. (i + 1)] = {{["ltp"] = "ltp_ih"}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}}} + end + --]] + + local layerRepo = nerv.LayerRepo(layers, pr, global_conf) + nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) + return layerRepo +end + +--global_conf: table +--layerRepo: nerv.LayerRepo +--Returns: a nerv.TNN +function prepare_tnn(global_conf, layerRepo) + nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) + + --input: input_w, input_w, ... input_w_now, last_activation + local connections_t = { + {"<input>[1]", "selectL1[1]", 0}, + + --{"selectL1[1]", "recurrentL1[1]", 0}, + --{"recurrentL1[1]", "sigmoidL1[1]", 0}, + --{"sigmoidL1[1]", "combinerL1[1]", 0}, + --{"combinerL1[1]", "recurrentL1[2]", 1}, + + {"selectL1[1]", "gruL1[1]", 0}, + {"gruL1[1]", "combinerL1[1]", 0}, + {"combinerL1[1]", "gruL1[2]", 1}, + {"combinerL1[2]", "dropoutL1[1]", 0}, + + {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, + {"outputL[1]", "softmaxL[1]", 0}, + {"<input>[2]", "softmaxL[2]", 0}, + {"softmaxL[1]", "<output>[1]", 0} + } + + for l = 2, global_conf.layer_num do + table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "gruL"..l.."[1]", 0}) + table.insert(connections_t, {"gruL"..l.."[1]", "combinerL"..l.."[1]", 0}) + table.insert(connections_t, {"combinerL"..l.."[1]", "gruL"..l.."[2]", 1}) + table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) + end + + --[[ + printf("%s printing DAG connections:\n", global_conf.sche_log_pre) + for key, value in pairs(connections_t) do + printf("\t%s->%s\n", key, value) + end + ]]-- + + local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, + ["dim_out"] = {1}, ["sub_layers"] = layerRepo, + ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, + }) + + tnn:init(global_conf.batch_size, global_conf.chunk_size) + + nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) + return tnn +end + +function prepare_dagL(global_conf, layerRepo) + nerv.printf("%s Generate and initing dagL ...\n", global_conf.sche_log_pre) + + --input: input_w, input_w, ... input_w_now, last_activation + local connections_t = { + ["<input>[1]"] = "selectL1[1]", + + ["selectL1[1]"] = "gruL1[1]", + ["gruL1[1]"] = "combinerL1[1]", + ["<input>[2]"] = "gruL1[2]", + --{"combinerL1[2]", "dropoutL1[1]", 0}, + + ["combinerL" .. global_conf.layer_num .. "[1]"] = "outputL[1]", + ["outputL[1]"] = "<output>[1]", + ["combinerL1[2]"] = "<output>[2]", + } + + if global_conf.layer_num > 1 then + nerv.error("multiple layer is currently not supported(not hard to implement though)") + end + --[[ + for l = 2, global_conf.layer_num do + table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "gruL"..l.."[1]", 0}) + table.insert(connections_t, {"gruL"..l.."[1]", "combinerL"..l.."[1]", 0}) + table.insert(connections_t, {"combinerL"..l.."[1]", "gruL"..l.."[2]", 1}) + table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) + end + ]]-- + + --[[ + printf("%s printing DAG connections:\n", global_conf.sche_log_pre) + for key, value in pairs(connections_t) do + printf("\t%s->%s\n", key, value) + end + ]]-- + + local dagL = nerv.DAGLayerT("dagL", global_conf, {["dim_in"] = {1, global_conf.hidden_size}, + ["dim_out"] = {global_conf.vocab:size(), global_conf.hidden_size}, ["sub_layers"] = layerRepo, + ["connections"] = connections_t + }) + + dagL:init(global_conf.batch_size) + + nerv.printf("%s Initing DAGL end.\n", global_conf.sche_log_pre) + return dagL +end + +function load_net_tnn(global_conf, fn) + prepare_parameters(global_conf, fn) + local layerRepo = prepare_layers(global_conf) + local tnn = prepare_tnn(global_conf, layerRepo) + return tnn +end + +function load_net_dagL(global_conf, fn) + prepare_parameters(global_conf, fn) + local layerRepo = prepare_layers(global_conf) + local dagL = prepare_dagL(global_conf, layerRepo) + return dagL +end + +local train_fn, valid_fn, test_fn +global_conf = {} +local set = arg[1] --"test" + +root_dir = '/home/slhome/txh18/workspace' + +if (set == "ptb") then + +data_dir = root_dir .. '/ptb/DATA' +train_fn = data_dir .. '/ptb.train.txt.adds' +valid_fn = data_dir .. '/ptb.valid.txt.adds' +test_fn = data_dir .. '/ptb.test.txt.adds' +vocab_fn = data_dir .. '/vocab' + +qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 32, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0.5", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/ptb/EXP-nerv/grulm_v1.0', + + fn_to_sample = root_dir .. '/ptb/EXP-nerv/grulm_v1.0h300l1ch15ba32slr0.15wc1e-05dr0.5/params.final', +} + +elseif (set == "msr_sc") then + +data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' +train_fn = data_dir .. '/normed_all.sf.len60.adds.train' +valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' +test_fn = data_dir .. '/answer_normed.adds' +vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' + +global_conf = { + lrate = 1, wcost = 1e-6, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 10, + max_iter = 30, + decay_iter = 10, + lr_decay = 1.003, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 400000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' +} + +elseif (set == "twitter") then + +data_dir = root_dir .. '/twitter_new/DATA' +train_fn = data_dir .. '/twitter.choose2.adds' +valid_fn = data_dir .. '/twitter.valid.adds' +test_fn = data_dir .. '/comm.test.choose-ppl.adds' +vocab_fn = data_dir .. '/twitter.choose.train.vocab' + +--qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 32, + max_iter = 30, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0.5", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 32, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/twitter_new/EXP-nerv/grulm_v1.0' +} + +else + +valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' + +global_conf = { + lrate = 0.01, wcost = 1e-5, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 20, + layer_num = 1, + chunk_size = 2, + batch_size = 10, + max_iter = 3, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + max_sen_len = 80, + lr_decay = 1.003, + decay_iter = 10, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 10, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_lstmlm_test' +} + +end + +lr_half = false --can not be local, to be set by loadstring +start_iter = -1 +start_lr = nil +ppl_last = 100000 +commands_str = "sampling" --"train:test" +commands = {} +test_iter = -1 +--for testout(question) +q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds" + +if arg[2] ~= nil then + nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) + loadstring(arg[2])() + nerv.LMUtil.wait(0.5) +else + nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) +end + +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str +global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' +global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' +global_conf.param_fn = global_conf.work_dir .. "/params" +global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) +global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') +commands = nerv.SUtil.parse_commands_set(commands_str) + +if start_lr ~= nil then + global_conf.lrate = start_lr +end + +--[[ +--redirecting log outputs! +nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) +]]-- + +----------------printing options--------------------------------- +nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) +for id, value in pairs(global_conf) do + nerv.printf("%s:\t%s\n", id, tostring(value)) +end +nerv.LMUtil.wait(2) + +nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) +nerv.printf("lr_half:\t%s\n", tostring(lr_half)) +nerv.printf("start_iter:\t%s\n", tostring(start_iter)) +nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) +nerv.printf("commands_str:\t%s\n", commands_str) +nerv.printf("test_iter:\t%s\n", tostring(test_iter)) +nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) +nerv.LMUtil.wait(2) +------------------printing options end------------------------------ + +math.randomseed(1) + +local vocab = nerv.LMVocab() +global_conf["vocab"] = vocab +nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) +global_conf.vocab:build_file(global_conf.vocab_fn, false) +ppl_rec = {} + +local final_iter = -1 +if commands["test"] == 1 then + nerv.printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + local tnn = load_net_tnn(global_conf, global_conf.fn_to_sample) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["test"] + +if commands["sampling"] == 1 then + nerv.printf("===SAMPLE===\n") + global_conf.sche_log_pre = "[SCHEDULER SAMPLING]:" + local dagL = load_net_dagL(global_conf, global_conf.fn_to_sample) + local sampler = nerv.LMSampler(global_conf) + sampler:load_dagL(dagL) + for k = 1, 5 do + local res = sampler:lm_sample_rnn_dagL(10, {}) + for i = 1, #res do + for j = 1, #res[i] do + nerv.printf("%s ", res[i][j].w) + end + nerv.printf("\n") + end + end + --global_conf.dropout_rate = 0 + --LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["sampling"] + + diff --git a/nerv/examples/mmi_chime3.lua b/nerv/examples/mmi_chime3.lua index 6ac7f28..3daaafa 100644 --- a/nerv/examples/mmi_chime3.lua +++ b/nerv/examples/mmi_chime3.lua @@ -160,6 +160,7 @@ function make_readers(feature_rspecifier, layer_repo) feature_rspecifier = feature_rspecifier, frm_ext = gconf.frm_ext, global_transf = layer_repo:get_layer("global_transf"), + need_key = true, mlfs = {} }) } diff --git a/nerv/examples/mpe_chime3.lua b/nerv/examples/mpe_chime3.lua index ec095b0..f9a2855 100644 --- a/nerv/examples/mpe_chime3.lua +++ b/nerv/examples/mpe_chime3.lua @@ -7,6 +7,8 @@ gconf = {lrate = 0.00001, wcost = 0, momentum = 0.0, tr_scp = "ark,s,cs:/slfs6/users/ymz09/kaldi/src/featbin/copy-feats scp:/slfs5/users/ymz09/chime/baseline/ASR/exp/tri4a_dnn_tr05_multi_enhanced_smbr/train.scp ark:- |", initialized_param = {"/slfs6/users/ymz09/nerv-project/nerv/nerv-speech/kaldi_seq/test/chime3_init.nerv", "/slfs6/users/ymz09/nerv-project/nerv/nerv-speech/kaldi_seq/test/chime3_global_transf.nerv"}, + decode_param = {"/slfs6/users/ymz09/nerv-project/test_mpe/1.nerv", + "/slfs6/users/ymz09/nerv-project/nerv/nerv-speech/kaldi_seq/test/chime3_global_transf.nerv"}, debug = false} function make_layer_repo(param_repo) @@ -125,13 +127,12 @@ function make_layer_repo(param_repo) ["mpe_crit[1]"] = "<output>[1]" } }}, - softmax_output = {{}, { + decode_output = {{}, { dim_in = {440}, dim_out = {2011}, sub_layers = layer_repo, connections = { ["<input>[1]"] = "main[1]", - ["main[1]"] = "softmax[1]", - ["softmax[1]"] = "<output>[1]" + ["main[1]"] = "<output>[1]" } }} } @@ -145,7 +146,7 @@ function get_network(layer_repo) end function get_decode_network(layer_repo) - return layer_repo:get_layer("softmax_output") + return layer_repo:get_layer("decode_output") end function get_global_transf(layer_repo) @@ -160,6 +161,7 @@ function make_readers(feature_rspecifier, layer_repo) feature_rspecifier = feature_rspecifier, frm_ext = gconf.frm_ext, global_transf = layer_repo:get_layer("global_transf"), + need_key = true, mlfs = {} }) } diff --git a/nerv/examples/swb_baseline.lua b/nerv/examples/swb_baseline.lua index 8f72200..51052ba 100644 --- a/nerv/examples/swb_baseline.lua +++ b/nerv/examples/swb_baseline.lua @@ -2,9 +2,9 @@ require 'htk_io' gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9, cumat_type = nerv.CuMatrixFloat, mmat_type = nerv.MMatrixFloat, - direct_update = true, + rearrange = true, -- just to make the context order consistent with old results, deprecated frm_ext = 5, - frm_trim = 5, + frm_trim = 5, -- trim the first and last 5 frames, TNet just does this, deprecated tr_scp = "/slfs1/users/mfy43/swb_ivec/train_bp.scp", cv_scp = "/slfs1/users/mfy43/swb_ivec/train_cv.scp", htk_conf = "/slfs1/users/mfy43/swb_ivec/plp_0_d_a.conf", diff --git a/nerv/layer/affine_recurrent.lua b/nerv/layer/affine_recurrent.lua index d537f4a..fd6f38f 100644 --- a/nerv/layer/affine_recurrent.lua +++ b/nerv/layer/affine_recurrent.lua @@ -9,31 +9,37 @@ function Recurrent:__init(id, global_conf, layer_conf) self.dim_in = layer_conf.dim_in self.dim_out = layer_conf.dim_out self.gconf = global_conf + self.log_pre = self.id .. "[LOG]" self.bp = self:find_param("bp", layer_conf, global_conf, nerv.BiasParam, {1, self.dim_out[1]}) --layer_conf.bp - self.ltp_hh = self:find_param("ltp_hh", layer_conf, global_conf, nerv.LinearTransParam, {self.dim_in[2], self.dim_out[1]}) --layer_conf.ltp_hh --from hidden to hidden + self.ltp_hh = self:find_param("ltphh", layer_conf, global_conf, nerv.LinearTransParam, {self.dim_in[2], self.dim_out[1]}) --layer_conf.ltp_hh --from hidden to hidden + self.ltp_ih = self:find_param("ltpih", layer_conf, global_conf, nerv.LinearTransParam, {self.dim_in[1], self.dim_out[1]}) --layer_conf.ltp_hh --from hidden to hidden self:check_dim_len(2, 1) self.direct_update = layer_conf.direct_update self.clip = layer_conf.clip --clip error in back_propagate + if self.clip ~= nil then + nerv.info("%s creating, will clip the error by %f", self.log_pre, self.clip) + end end --Check parameter function Recurrent:init(batch_size) - if (self.ltp_hh.trans:ncol() ~= self.bp.trans:ncol()) then + if self.ltp_hh.trans:ncol() ~= self.bp.trans:ncol() or + self.ltp_ih.trans:ncol() ~= self.bp.trans:ncol() then nerv.error("mismatching dimensions of ltp and bp") end - if (self.dim_in[1] ~= self.ltp_hh.trans:nrow() or - self.dim_in[2] ~= self.ltp_hh.trans:nrow()) then + if self.dim_in[1] ~= self.ltp_ih.trans:nrow() or + self.dim_in[2] ~= self.ltp_hh.trans:nrow() then nerv.error("mismatching dimensions of ltp and input") end if (self.dim_out[1] ~= self.bp.trans:ncol()) then nerv.error("mismatching dimensions of bp and output") end - self.ltp_hh_grad = self.ltp_hh.trans:create() self.ltp_hh:train_init() + self.ltp_ih:train_init() self.bp:train_init() end @@ -42,39 +48,19 @@ function Recurrent:batch_resize(batch_size) end function Recurrent:update(bp_err, input, output) - if self.direct_update == true then - local ltp_hh = self.ltp_hh.trans - local bp = self.bp.trans - local gconf = self.gconf - if (gconf.momentum > 0) then - -- momentum gain - local mmt_gain = 1.0 / (1.0 - gconf.momentum) - local n = input[1]:nrow() * mmt_gain - -- update corrections (accumulated errors) - self.ltp_hh.correction:mul(input[2], bp_err[1], 1.0, gconf.momentum, 'T', 'N') - self.bp.correction:add(self.bp.correction, bp_err[1]:colsum(), gconf.momentum, 1.0) - -- perform update and weight decay - ltp_hh:add(ltp_hh, self.ltp_hh.correction, 1.0 - gconf.lrate * gconf.wcost / gconf.batch_size, - gconf.lrate / n) - bp:add(bp, self.bp.correction, 1.0 - gconf.lrate * gconf.wcost / gconf.batch_size, - gconf.lrate / n) - else - ltp_hh:mul(input[2], bp_err[1], - gconf.lrate / gconf.batch_size, 1.0 - gconf.wcost * gconf.lrate / gconf.batch_size, 'T', 'N') - bp:add(bp, bp_err[1]:colsum(), 1.0 - gconf.lrate * gconf.wcost / gconf.batch_size, - gconf.lrate / gconf.batch_size) - end - else - --self.ltp_hh_grad:mul(input[2], bp_err[1], 1.0, 0.0, 'T', 'N') - self.ltp_hh:update_by_err_input(bp_err[1], input[2]) - self.bp:update_by_gradient(bp_err[1]:colsum()) - end + self.ltp_ih:update_by_err_input(bp_err[1], input[1]) + self.ltp_hh:update_by_err_input(bp_err[1], input[2]) + self.bp:update_by_gradient(bp_err[1]:colsum()) end function Recurrent:propagate(input, output) - output[1]:copy_fromd(input[1]) + output[1]:mul(input[1], self.ltp_ih.trans, 1.0, 0.0, 'N', 'N') output[1]:mul(input[2], self.ltp_hh.trans, 1.0, 1.0, 'N', 'N') output[1]:add_row(self.bp.trans, 1.0) end function Recurrent:back_propagate(bp_err, next_bp_err, input, output) - next_bp_err[1]:copy_fromd(bp_err[1]) + next_bp_err[1]:mul(bp_err[1], self.ltp_ih.trans, 1.0, 0.0, 'N', 'T') next_bp_err[2]:mul(bp_err[1], self.ltp_hh.trans, 1.0, 0.0, 'N', 'T') --[[ for i = 0, next_bp_err[2]:nrow() - 1 do @@ -84,11 +70,11 @@ function Recurrent:back_propagate(bp_err, next_bp_err, input, output) end end ]]-- - if (self.clip ~= nil) then - next_bp_err[2]:clip(- self.clip, self.clip) + if self.clip ~= nil then + next_bp_err[2]:clip(-self.clip, self.clip) end end function Recurrent:get_params() - return nerv.ParamRepo({self.ltp_hh, self.bp}) + return nerv.ParamRepo({self.ltp_ih, self.ltp_hh, self.bp}) end diff --git a/nerv/lib/matrix/cukernel.cu b/nerv/lib/matrix/cukernel.cu index 210e6bf..c20e538 100644 --- a/nerv/lib/matrix/cukernel.cu +++ b/nerv/lib/matrix/cukernel.cu @@ -44,6 +44,7 @@ __device__ float atomicAdd_nvidia(float* address, float val) { #undef MATRIX_USE_FLOAT #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR +#undef MATRIX_ELEM_PTR_BASE #undef MATRIX_ELEM_FMT #undef MATRIX_ELEM_WRITE_FMT diff --git a/nerv/lib/matrix/cumatrix.c b/nerv/lib/matrix/cumatrix.c index ff1168d..a8ed075 100644 --- a/nerv/lib/matrix/cumatrix.c +++ b/nerv/lib/matrix/cumatrix.c @@ -57,6 +57,7 @@ void nerv_cumatrix_init() { #undef MATRIX_USE_FLOAT #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR +#undef MATRIX_ELEM_PTR_BASE #undef MATRIX_ELEM_FMT #undef MATRIX_ELEM_WRITE_FMT #undef MATRIX_CUMATRIX_HOST_TNAME diff --git a/nerv/lib/matrix/generic/cukernel.cu b/nerv/lib/matrix/generic/cukernel.cu index aa830b5..2b696d5 100644 --- a/nerv/lib/matrix/generic/cukernel.cu +++ b/nerv/lib/matrix/generic/cukernel.cu @@ -262,12 +262,29 @@ __global__ void cudak_(clip)(MATRIX_ELEM *a, } #ifdef __NERV_FUTURE_CUDA_7 -__global__ void cudak_(update_select_rows)(MATRIX_ELEM *c, const MATRIX_ELEM *a, const MATRIX_ELEM *idx, - int nrow_a, int ncol_a, int stride_c, int stride_a, double alpha, double beta) { +__global__ void cudak_(update_select_rows_by_rowidx)(MATRIX_ELEM *c, const MATRIX_ELEM *a, const MATRIX_ELEM *idx, + int nrow_a, int ncol_a, int nrow_c, int stride_c, int stride_a, double alpha, double beta) { int j = blockIdx.x * blockDim.x + threadIdx.x; int i = blockIdx.y * blockDim.y + threadIdx.y; if (i >= nrow_a || j >= ncol_a) return; int i_c = lrintf(idx[i]); + if (i_c < 0 || i_c >= nrow_c) { + printf("ERROR inside kernel update_select_rows, i_c(%d) out of range!", i_c); + } + //critical: i_c could conflict among threads(same index in the idx array), so atomicAdd is used + //c[j + i_c * stride_c] = c[j + i_c * stride_c] * (1 - beta * alpha) + a[j + i * stride_a] * alpha; + atomicAdd_nvidia(c + j + i_c * stride_c, c[j + i_c * stride_c] * (- beta * alpha) + a[j + i * stride_a] * alpha); +} + +__global__ void cudak_(update_select_rows_by_colidx)(MATRIX_ELEM *c, const MATRIX_ELEM *a, const MATRIX_ELEM *idx, + int nrow_a, int ncol_a, int nrow_c, int stride_c, int stride_a, int stride_idx, double alpha, double beta) { + int j = blockIdx.x * blockDim.x + threadIdx.x; + int i = blockIdx.y * blockDim.y + threadIdx.y; + if (i >= nrow_a || j >= ncol_a) return; + int i_c = lrintf(idx[stride_idx * i]); + if (i_c < 0 || i_c >= nrow_c) { + printf("ERROR inside kernel update_select_rows, i_c(%d) out of range!", i_c); + } //critical: i_c could conflict among threads(same index in the idx array), so atomicAdd is used //c[j + i_c * stride_c] = c[j + i_c * stride_c] * (1 - beta * alpha) + a[j + i * stride_a] * alpha; atomicAdd_nvidia(c + j + i_c * stride_c, c[j + i_c * stride_c] * (- beta * alpha) + a[j + i * stride_a] * alpha); @@ -335,13 +352,31 @@ __global__ void cudak_(gen_col_idx)(MATRIX_ELEM *b, __global__ void cudak_(copy_rows_by_idx)(const MATRIX_ELEM *a, MATRIX_ELEM *b, const MATRIX_ELEM *idx, - int nrow, int ncol, int stride) { + int nrow, int ncol, int a_nrow, int stride) { + int j = blockIdx.x * blockDim.x + threadIdx.x; + int i = blockIdx.y * blockDim.y + threadIdx.y; + if (i >= nrow || j >= ncol) return; + int k = lrintf(idx[i]); + if (k < 0 || k >= a_nrow) { + printf("error in kernel copy_rows_by_idx k(%d) out of range\n", k); + } + b[j + i * stride] = a[j + k * stride]; +} + +__global__ void cudak_(copy_rows_by_colidx)(const MATRIX_ELEM *a, MATRIX_ELEM *b, + const MATRIX_ELEM *idx, + int nrow, int ncol, int a_nrow, int stride, int idx_stride) { int j = blockIdx.x * blockDim.x + threadIdx.x; int i = blockIdx.y * blockDim.y + threadIdx.y; if (i >= nrow || j >= ncol) return; - b[j + i * stride] = a[j + lrintf(idx[i]) * stride]; + int k = lrintf(idx[i * idx_stride]); + if (k < 0 || k >= a_nrow) { + printf("error in kernel copy_rows_by_colidx k(%d) out of range\n", k); + } + b[j + i * stride] = a[j + k * stride]; } + extern "C" { #include "../cukernel.h" void cudak_(cuda_log_elem)(const Matrix *a, Matrix *b) { @@ -633,16 +668,26 @@ extern "C" { } #ifdef __NERV_FUTURE_CUDA_7 - void cudak_(cuda_update_select_rows)(Matrix *c, const Matrix *a, const Matrix *idx, double alpha, double beta) { + void cudak_(cuda_update_select_rows_by_rowidx)(Matrix *c, const Matrix *a, const Matrix *idx, double alpha, double beta) { dim3 threadsPerBlock(CUDA_THREADS_N, CUDA_THREADS_N); dim3 numBlocks(CEIL_DIV(a->ncol, threadsPerBlock.x), CEIL_DIV(a->nrow, threadsPerBlock.y)); - cudak_(update_select_rows)<<<numBlocks, threadsPerBlock>>> \ + cudak_(update_select_rows_by_rowidx)<<<numBlocks, threadsPerBlock>>> \ (MATRIX_ELEM_PTR(c), MATRIX_ELEM_PTR(a), MATRIX_ELEM_PTR(idx), - a->nrow, a->ncol, c->stride / sizeof(MATRIX_ELEM), + a->nrow, a->ncol, c->nrow, c->stride / sizeof(MATRIX_ELEM), a->stride / sizeof(MATRIX_ELEM), alpha, beta); cudaStreamSynchronize(0); } + void cudak_(cuda_update_select_rows_by_colidx)(Matrix *c, const Matrix *a, const Matrix *idx, double alpha, double beta) { + dim3 threadsPerBlock(CUDA_THREADS_N, CUDA_THREADS_N); + dim3 numBlocks(CEIL_DIV(a->ncol, threadsPerBlock.x), + CEIL_DIV(a->nrow, threadsPerBlock.y)); + cudak_(update_select_rows_by_colidx)<<<numBlocks, threadsPerBlock>>> \ + (MATRIX_ELEM_PTR(c), MATRIX_ELEM_PTR(a), MATRIX_ELEM_PTR(idx), + a->nrow, a->ncol, c->nrow, c->stride / sizeof(MATRIX_ELEM), + a->stride / sizeof(MATRIX_ELEM), idx->stride / sizeof(MATRIX_ELEM), alpha, beta); + cudaStreamSynchronize(0); + } #endif void cudak_(cuda_expand_frm)(const Matrix *a, Matrix *b, int context) { @@ -710,7 +755,18 @@ extern "C" { cudak_(copy_rows_by_idx)<<<numBlocks, threadsPerBlock>>> \ (MATRIX_ELEM_PTR(a), MATRIX_ELEM_PTR(b), MATRIX_ELEM_PTR(idx) + idx_begin, - b->nrow, b->ncol, b->stride / sizeof(MATRIX_ELEM)); + b->nrow, b->ncol, a->nrow, b->stride / sizeof(MATRIX_ELEM)); + cudaStreamSynchronize(0); + } + + void cudak_(cuda_copy_rows_by_colidx)(const Matrix *a, Matrix *b, + const Matrix *idx, int idx_begin) { + dim3 threadsPerBlock(CUDA_THREADS_NN, 1); + dim3 numBlocks(CEIL_DIV(b->ncol, threadsPerBlock.x), b->nrow); + cudak_(copy_rows_by_colidx)<<<numBlocks, threadsPerBlock>>> \ + (MATRIX_ELEM_PTR(a), MATRIX_ELEM_PTR(b), + MATRIX_ELEM_PTR(idx) + idx_begin, + b->nrow, b->ncol, a->nrow, b->stride / sizeof(MATRIX_ELEM), idx->stride / sizeof(MATRIX_ELEM)); cudaStreamSynchronize(0); } } diff --git a/nerv/lib/matrix/generic/cumatrix.c b/nerv/lib/matrix/generic/cumatrix.c index 65e0788..7582725 100644 --- a/nerv/lib/matrix/generic/cumatrix.c +++ b/nerv/lib/matrix/generic/cumatrix.c @@ -349,7 +349,7 @@ void nerv_matrix_(copy_rows_fromh_by_idx)(Matrix *a, const Matrix *b, long nrow = a->nrow; if (!(0 <= b_begin && b_begin + nrow <= idx->ncol)) NERV_EXIT_STATUS(status, MAT_INVALID_COPY_INTERVAL, 0); - float *idx_ptr = idx->data.f; + float *idx_ptr = MATRIX_ELEM_PTR_F(idx); int i; if (idx->nrow != 1) NERV_EXIT_STATUS(status, MAT_IDX_VECTOR_EXP, 0); @@ -393,15 +393,45 @@ void nerv_matrix_(copy_rows_fromd_by_idx)(Matrix *a, const Matrix *b, NERV_SET_STATUS(status, NERV_NORMAL, 0); } +void nerv_matrix_(copy_rows_fromd_by_colidx)(Matrix *a, const Matrix *b, + const Matrix *idx, int b_begin, Status *status) { + long nrow = a->nrow; + if (!(0 <= b_begin && b_begin + nrow <= idx->nrow)) + NERV_EXIT_STATUS(status, MAT_INVALID_COPY_INTERVAL, 0); + if (idx->ncol != 1) + NERV_EXIT_STATUS(status, MAT_IDX_VECTOR_EXP, 0); + if (a->ncol != b->ncol) { + printf("%d %d\n", a->ncol, b->ncol); + NERV_EXIT_STATUS(status, MAT_MISMATCH_DIM, 0); + } + PROFILE_START + cudak_(cuda_copy_rows_by_colidx)(b, a, idx, b_begin); + PROFILE_STOP + NERV_SET_STATUS(status, NERV_NORMAL, 0); +} + + #ifdef __NERV_FUTURE_CUDA_7 -void nerv_matrix_(update_select_rows)(Matrix *c, const Matrix *a, const Matrix *idx, double alpha, double beta, Status *status) { +void nerv_matrix_(update_select_rows_by_rowidx)(Matrix *c, const Matrix *a, const Matrix *idx, double alpha, double beta, Status *status) { long nrow = a->nrow; - if (idx->nrow != 1) + if (idx->nrow != 1 || idx->ncol != a->nrow) + NERV_EXIT_STATUS(status, MAT_IDX_VECTOR_EXP, 0); + if (a->ncol != c->ncol) + NERV_EXIT_STATUS(status, MAT_MISMATCH_DIM, 0); + PROFILE_START + cudak_(cuda_update_select_rows_by_rowidx)(c, a, idx, alpha, beta); + PROFILE_STOP + NERV_SET_STATUS(status, NERV_NORMAL, 0); +} + +void nerv_matrix_(update_select_rows_by_colidx)(Matrix *c, const Matrix *a, const Matrix *idx, double alpha, double beta, Status *status) { + long nrow = a->nrow; + if (idx->ncol != 1 || idx->nrow != a->nrow) NERV_EXIT_STATUS(status, MAT_IDX_VECTOR_EXP, 0); if (a->ncol != c->ncol) NERV_EXIT_STATUS(status, MAT_MISMATCH_DIM, 0); PROFILE_START - cudak_(cuda_update_select_rows)(c, a, idx, alpha, beta); + cudak_(cuda_update_select_rows_by_colidx)(c, a, idx, alpha, beta); PROFILE_STOP NERV_SET_STATUS(status, NERV_NORMAL, 0); } diff --git a/nerv/lib/matrix/generic/cumatrix.h b/nerv/lib/matrix/generic/cumatrix.h index aa8805a..e82dccd 100644 --- a/nerv/lib/matrix/generic/cumatrix.h +++ b/nerv/lib/matrix/generic/cumatrix.h @@ -45,7 +45,13 @@ void nerv_matrix_(copy_rows_fromh_by_idx)(Matrix *a, const Matrix *b, const Matrix *idx, int b_begin, Status *status); void nerv_matrix_(copy_rows_fromd_by_idx)(Matrix *a, const Matrix *b, const Matrix *idx, int b_begin, Status *status); -void nerv_matrix_(update_select_rows)(Matrix *c, const Matrix *a, const Matrix *idx, double alpha, double beta, Status *status); +void nerv_matrix_(copy_rows_fromd_by_colidx)(Matrix *a, const Matrix *b, + const Matrix *idx, int b_begin, Status *status); + +#ifdef __NERV_FUTURE_CUDA_7 +void nerv_matrix_(update_select_rows_by_rowidx)(Matrix *c, const Matrix *a, const Matrix *idx, double alpha, double beta, Status *status); +void nerv_matrix_(update_select_rows_by_colidx)(Matrix *c, const Matrix *a, const Matrix *idx, double alpha, double beta, Status *status); +#endif void nerv_matrix_(expand_frm)(Matrix *a, const Matrix *b, int context, Status *status); diff --git a/nerv/lib/matrix/generic/elem_type.h b/nerv/lib/matrix/generic/elem_type.h index bffe940..07f6355 100644 --- a/nerv/lib/matrix/generic/elem_type.h +++ b/nerv/lib/matrix/generic/elem_type.h @@ -1,22 +1,29 @@ +#define MATRIX_ELEM_PTR_F(self) ((float *)((char *)((self)->data.f) + (self)->offset)) +#define MATRIX_ELEM_PTR_D(self) ((double *)((char *)((self)->data.d) + (self)->offset)) +#define MATRIX_ELEM_PTR_I(self) ((long *)((char *)((self)->data.i) + (self)->offset)) + #ifdef MATRIX_USE_FLOAT #define MATRIX_ELEM float #define MATRIX_ELEM_FMT "%f" #define MATRIX_ELEM_WRITE_FMT "%.8f" -#define MATRIX_ELEM_PTR(self) ((self)->data.f) +#define MATRIX_ELEM_PTR(self) MATRIX_ELEM_PTR_F(self) +#define MATRIX_ELEM_PTR_BASE(self) ((self)->data.f) #elif defined(MATRIX_USE_DOUBLE) #define MATRIX_ELEM double #define MATRIX_ELEM_FMT "%lf" #define MATRIX_ELEM_WRITE_FMT "%.8lf" -#define MATRIX_ELEM_PTR(self) ((self)->data.d) +#define MATRIX_ELEM_PTR(self) MATRIX_ELEM_PTR_D(self) +#define MATRIX_ELEM_PTR_BASE(self) ((self)->data.d) #elif defined(MATRIX_USE_INT) #define MATRIX_ELEM long #define MATRIX_ELEM_FMT "%ld" #define MATRIX_ELEM_WRITE_FMT "%ld" -#define MATRIX_ELEM_PTR(self) ((self)->data.i) +#define MATRIX_ELEM_PTR(self) MATRIX_ELEM_PTR_I(self) +#define MATRIX_ELEM_PTR_BASE(self) ((self)->data.i) #endif diff --git a/nerv/lib/matrix/generic/matrix.c b/nerv/lib/matrix/generic/matrix.c index fd5d28f..004d9aa 100644 --- a/nerv/lib/matrix/generic/matrix.c +++ b/nerv/lib/matrix/generic/matrix.c @@ -4,12 +4,11 @@ /* FIXME: malloc failure detection */ void nerv_matrix_(data_free)(Matrix *self, Status *status) { - if(*self->data_ref == 0) return; /* FIXME: repeat free memory */ assert(*self->data_ref > 0); if (--(*self->data_ref) == 0) { /* free matrix data */ - MATRIX_DATA_FREE(MATRIX_ELEM_PTR(self), status); + MATRIX_DATA_FREE(MATRIX_ELEM_PTR_BASE(self), status); curandDestroyGenerator(*(self->curand_gen)); free(self->curand_gen); free(self->data_ref); @@ -31,7 +30,7 @@ Matrix *nerv_matrix_(create)(long nrow, long ncol, Status *status) { self->ncol = ncol; self->nmax = self->nrow * self->ncol; self->dim = 2; - MATRIX_DATA_ALLOC(&MATRIX_ELEM_PTR(self), &self->stride, + MATRIX_DATA_ALLOC(&MATRIX_ELEM_PTR_BASE(self), &self->stride, sizeof(MATRIX_ELEM) * self->ncol, self->nrow, status); if (status->err_code != NERV_NORMAL) @@ -46,6 +45,7 @@ Matrix *nerv_matrix_(create)(long nrow, long ncol, Status *status) { curandCreateGenerator(self->curand_gen, CURAND_RNG_PSEUDO_DEFAULT); curandSetPseudoRandomGeneratorSeed(*(self->curand_gen), time(NULL)); + self->offset = 0; nerv_matrix_(data_retain)(self); NERV_SET_STATUS(status, NERV_NORMAL, 0); return self; @@ -62,9 +62,10 @@ Matrix *nerv_matrix_(getrow)(Matrix *self, int row) { prow->dim = 1; prow->stride = self->stride; prow->nmax = prow->ncol; - MATRIX_ELEM_PTR(prow) = MATRIX_ROW_PTR(self, row); + prow->data = self->data; prow->data_ref = self->data_ref; prow->curand_gen = self->curand_gen; + prow->offset = row * self->stride; nerv_matrix_(data_retain)(prow); return prow; } diff --git a/nerv/lib/matrix/matrix.h b/nerv/lib/matrix/matrix.h index 5a85c08..a28fd97 100644 --- a/nerv/lib/matrix/matrix.h +++ b/nerv/lib/matrix/matrix.h @@ -13,6 +13,7 @@ typedef struct Matrix { double *d; long *i; } data; /* pointer to actual storage */ + unsigned long offset; /* the actual beginning of the matrix */ long *data_ref; curandGenerator_t *curand_gen; } Matrix; diff --git a/nerv/lib/matrix/mmatrix.c b/nerv/lib/matrix/mmatrix.c index b8157eb..b5670f2 100644 --- a/nerv/lib/matrix/mmatrix.c +++ b/nerv/lib/matrix/mmatrix.c @@ -6,6 +6,7 @@ #define host_matrix_(NAME) host_matrix_float_##NAME #define nerv_matrix_(NAME) nerv_matrix_host_float_##NAME #include "generic/matrix.h" +#include "generic/elem_type.h" #include "generic/mmatrix.c" Matrix *nerv_matrix_(perm_gen)(int ncol, Status *status) { @@ -13,7 +14,7 @@ Matrix *nerv_matrix_(perm_gen)(int ncol, Status *status) { Matrix *self = nerv_matrix_(create)(1, ncol, status); if (status->err_code != NERV_NORMAL) return NULL; - float *prow = self->data.f; + float *prow = MATRIX_ELEM_PTR_F(self); for (i = 0; i < ncol; i++) prow[i] = i; for (i = ncol - 1; i >= 0; i--) @@ -31,6 +32,7 @@ Matrix *nerv_matrix_(perm_gen)(int ncol, Status *status) { #undef MATRIX_USE_FLOAT #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR +#undef MATRIX_ELEM_PTR_BASE #undef MATRIX_ELEM_FMT #undef MATRIX_ELEM_WRITE_FMT @@ -44,6 +46,7 @@ Matrix *nerv_matrix_(perm_gen)(int ncol, Status *status) { #undef MATRIX_USE_DOUBLE #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR +#undef MATRIX_ELEM_PTR_BASE #undef MATRIX_ELEM_FMT #undef MATRIX_ELEM_WRITE_FMT diff --git a/nerv/matrix/cumatrix.c b/nerv/matrix/cumatrix.c index fef03fc..bf92f92 100644 --- a/nerv/matrix/cumatrix.c +++ b/nerv/matrix/cumatrix.c @@ -49,6 +49,7 @@ const char *nerv_matrix_(tname) = "nerv.CuMatrixFloat"; #undef MATRIX_USE_FLOAT #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR +#undef MATRIX_ELEM_PTR_BASE #undef MATRIX_ELEM_FMT #undef MATRIX_ELEM_WRITE_FMT #undef MATRIX_CUMATRIX_HOST_TNAME diff --git a/nerv/matrix/generic/cumatrix.c b/nerv/matrix/generic/cumatrix.c index fb36033..edd7b0a 100644 --- a/nerv/matrix/generic/cumatrix.c +++ b/nerv/matrix/generic/cumatrix.c @@ -1,5 +1,5 @@ #ifdef NERV_GENERIC_CUMATRIX -#include "elem_type.h" +#include "../../lib/matrix/generic/elem_type.h" #define MATRIX_DATA_WRITE(L, data, idx, val) cuda_matrix_(write)(L, data, idx, val) #define MATRIX_DATA_READ(L, data, idx) cuda_matrix_(read)(L, data, idx) #define MATRIX_INIT(L) cuda_matrix_(init)(L) @@ -292,6 +292,18 @@ static int nerv_matrix_(lua_copy_rows_fromd_by_idx)(lua_State *L) { return 0; } +static int nerv_matrix_(lua_copy_rows_fromd_by_colidx)(lua_State *L) { + Status status; + Matrix *a = luaT_checkudata(L, 1, nerv_matrix_(tname)); + const Matrix *b = luaT_checkudata(L, 2, nerv_matrix_(tname)); + const Matrix *idx = luaT_checkudata(L, 3, nerv_matrix_(tname)); + long nrow = a->nrow; + int idx_begin = lua_gettop(L) > 3 ? luaL_checkinteger(L, 4) : 0; + nerv_matrix_(copy_rows_fromd_by_colidx)(a, b, idx, idx_begin, &status); + NERV_LUA_CHECK_STATUS(L, status); + return 0; +} + static int nerv_matrix_(lua_expand_frm)(lua_State *L) { Status status; Matrix *a = luaT_checkudata(L, 1, nerv_matrix_(tname)); @@ -331,7 +343,21 @@ static int nerv_matrix_(lua_scale_rows_by_row)(lua_State *L) { } #ifdef __NERV_FUTURE_CUDA_7 -static int nerv_matrix_(lua_update_select_rows)(lua_State *L) { +static int nerv_matrix_(lua_update_select_rows_by_rowidx)(lua_State *L) { + /* update c's select rows, + * i.e. c[idx[i]] = c[idx[i]] * (1 - beta * alpha) + a[i] * alpha */ + Status status; + Matrix *c = luaT_checkudata(L, 1, nerv_matrix_(tname)); + const Matrix *a = luaT_checkudata(L, 2, nerv_matrix_(tname)); + const Matrix *idx = luaT_checkudata(L, 3, nerv_matrix_(tname)); + MATRIX_ELEM alpha = luaL_checknumber(L, 4); + MATRIX_ELEM beta = luaL_checknumber(L, 5); + nerv_matrix_(update_select_rows_by_rowidx)(c, a, idx, alpha, beta, &status); + NERV_LUA_CHECK_STATUS(L, status); + return 0; +} + +static int nerv_matrix_(lua_update_select_rows_by_colidx)(lua_State *L) { /* update c's select rows, * i.e. c[idx[i]] = c[idx[i]] * (1 - beta * alpha) + a[i] * alpha */ Status status; @@ -340,7 +366,7 @@ static int nerv_matrix_(lua_update_select_rows)(lua_State *L) { const Matrix *idx = luaT_checkudata(L, 3, nerv_matrix_(tname)); MATRIX_ELEM alpha = luaL_checknumber(L, 4); MATRIX_ELEM beta = luaL_checknumber(L, 5); - nerv_matrix_(update_select_rows)(c, a, idx, alpha, beta, &status); + nerv_matrix_(update_select_rows_by_colidx)(c, a, idx, alpha, beta, &status); NERV_LUA_CHECK_STATUS(L, status); return 0; } @@ -376,12 +402,14 @@ static const luaL_Reg nerv_matrix_(extra_methods)[] = { {"thres_mask", nerv_matrix_(lua_thres_mask)}, {"copy_rows_fromh_by_idx", nerv_matrix_(lua_copy_rows_fromh_by_idx)}, {"copy_rows_fromd_by_idx", nerv_matrix_(lua_copy_rows_fromd_by_idx)}, + {"copy_rows_fromd_by_colidx", nerv_matrix_(lua_copy_rows_fromd_by_colidx)}, {"expand_frm", nerv_matrix_(lua_expand_frm)}, {"rearrange_frm", nerv_matrix_(lua_rearrange_frm)}, {"scale_rows_by_row", nerv_matrix_(lua_scale_rows_by_row)}, {"scale_rows_by_col", nerv_matrix_(lua_scale_rows_by_col)}, #ifdef __NERV_FUTURE_CUDA_7 - {"update_select_rows", nerv_matrix_(lua_update_select_rows)}, + {"update_select_rows_by_rowidx", nerv_matrix_(lua_update_select_rows_by_rowidx)}, + {"update_select_rows_by_colidx", nerv_matrix_(lua_update_select_rows_by_colidx)}, #endif {NULL, NULL} }; diff --git a/nerv/matrix/generic/elem_type.h b/nerv/matrix/generic/elem_type.h deleted file mode 100644 index bffe940..0000000 --- a/nerv/matrix/generic/elem_type.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifdef MATRIX_USE_FLOAT - -#define MATRIX_ELEM float -#define MATRIX_ELEM_FMT "%f" -#define MATRIX_ELEM_WRITE_FMT "%.8f" -#define MATRIX_ELEM_PTR(self) ((self)->data.f) - -#elif defined(MATRIX_USE_DOUBLE) - -#define MATRIX_ELEM double -#define MATRIX_ELEM_FMT "%lf" -#define MATRIX_ELEM_WRITE_FMT "%.8lf" -#define MATRIX_ELEM_PTR(self) ((self)->data.d) - -#elif defined(MATRIX_USE_INT) - -#define MATRIX_ELEM long -#define MATRIX_ELEM_FMT "%ld" -#define MATRIX_ELEM_WRITE_FMT "%ld" -#define MATRIX_ELEM_PTR(self) ((self)->data.i) - -#endif diff --git a/nerv/matrix/generic/mmatrix.c b/nerv/matrix/generic/mmatrix.c index 01dd9e5..a4e8489 100644 --- a/nerv/matrix/generic/mmatrix.c +++ b/nerv/matrix/generic/mmatrix.c @@ -1,6 +1,6 @@ #ifdef NERV_GENERIC_MMATRIX #include "../../lib/matrix/generic/matrix.h" -#include "elem_type.h" +#include "../../lib/matrix/generic/elem_type.h" #define MATRIX_DATA_WRITE(L, data, idx, val) (data[idx] = val) #define MATRIX_DATA_READ(L, data, idx) (data[idx]) #define MATRIX_INIT(L) host_matrix_(init)(L) diff --git a/nerv/matrix/mmatrix.c b/nerv/matrix/mmatrix.c index 961059c..20c31d6 100644 --- a/nerv/matrix/mmatrix.c +++ b/nerv/matrix/mmatrix.c @@ -45,6 +45,7 @@ static const luaL_Reg nerv_matrix_(extra_methods_int)[] = { #undef MATRIX_USE_FLOAT #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR +#undef MATRIX_ELEM_PTR_BASE #undef MATRIX_ELEM_FMT #undef MATRIX_ELEM_WRITE_FMT #undef MMATRIX_INIT @@ -60,6 +61,7 @@ const char *nerv_matrix_(tname) = "nerv.MMatrixDouble"; #undef MATRIX_USE_DOUBLE #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR +#undef MATRIX_ELEM_PTR_BASE #undef MATRIX_ELEM_FMT #undef MATRIX_ELEM_WRITE_FMT diff --git a/nerv/tnn/init.lua b/nerv/tnn/init.lua index 979f5d8..b375fa8 100644 --- a/nerv/tnn/init.lua +++ b/nerv/tnn/init.lua @@ -33,6 +33,8 @@ function LayerT:check_dim_len(len_in, len_out) end end +LayerT.find_param = nerv.Layer.find_param + function LayerT:get_params() nerv.error_method_not_implemented() end diff --git a/nerv/tnn/tnn.lua b/nerv/tnn/tnn.lua index cf02123..d527fe6 100644 --- a/nerv/tnn/tnn.lua +++ b/nerv/tnn/tnn.lua @@ -64,7 +64,7 @@ function TNN.make_initial_store(st, p, dim, batch_size, chunk_size, extend_t, gl if (type(st) ~= "table") then nerv.error("st should be a table") end - for i = 1 - extend_t - 1, chunk_size + extend_t + 1 do --intentionally allocated more time + for i = 1 - extend_t - 2, chunk_size + extend_t + 2 do --intentionally allocated more time if (st[i] == nil) then st[i] = {} end @@ -77,6 +77,7 @@ function TNN.make_initial_store(st, p, dim, batch_size, chunk_size, extend_t, gl st_c[i + t_c][p_c] = st[i][p] end end + collectgarbage("collect") --free the old one to save memory end function TNN:out_of_feedrange(t) --out of chunk, or no input, for the current feed @@ -111,6 +112,10 @@ function TNN:__init(id, global_conf, layer_conf) local dim_out = layer_conf.dim_out local parsed_conns = {} local _ + + for id, _ in pairs(layer_conf.sub_layers.layers) do --caution: with this line, some layer not connected will be included + discover(id, layers, layer_conf.sub_layers) + end for _, ll in pairs(layer_conf.connections) do local id_from, port_from = parse_id(ll[1]) @@ -183,7 +188,6 @@ function TNN:init(batch_size, chunk_size) self.make_initial_store(ref_from.outputs_m, port_from, dim, batch_size, chunk_size, self.extend_t, self.gconf, ref_to.inputs_m, port_to, time) ref_from.err_inputs_matbak_p[port_from] = self.gconf.cumat_type(batch_size, dim) self.make_initial_store(ref_from.err_inputs_m, port_from, dim, batch_size, chunk_size, self.extend_t, self.gconf, ref_to.err_outputs_m, port_to, time) - end self.outputs_m = {} @@ -216,7 +220,9 @@ function TNN:init(batch_size, chunk_size) end end -- initialize sub layers + nerv.info("TNN initing sub-layer %s", ref.id) ref.layer:init(batch_size, chunk_size) + collectgarbage("collect") end local flags_now = {} @@ -339,6 +345,13 @@ function TNN:net_propagate() --propagate according to feeds_now end local feeds_now = self.feeds_now + for t = 1, self.chunk_size do --some layer maybe do not have inputs from time 1..chunk_size + for id, ref in pairs(self.layers) do + if #ref.dim_in > 0 then --some layer is just there(only to save some parameter) + self:propagate_dfs(ref, t) + end + end + end for t = 1, self.chunk_size do if (bit.band(feeds_now.flagsPack_now[t], nerv.TNN.FC.HAS_INPUT) > 0) then for i = 1, #self.dim_in do @@ -362,6 +375,7 @@ function TNN:net_propagate() --propagate according to feeds_now end end end + if (flag_out == false) then nerv.error("some thing wrong, some labeled output is not propagated") end @@ -458,6 +472,13 @@ function TNN:net_backpropagate(do_update) --propagate according to feeds_now end local feeds_now = self.feeds_now + for t = 1, self.chunk_size do --some layer maybe do not have outputs from time 1..chunk_size + for id, ref in pairs(self.layers) do + if #ref.dim_out > 0 then --some layer is just there(only to save some parameter) + self:backpropagate_dfs(ref, t, do_update) + end + end + end for t = 1, self.chunk_size do if bit.band(feeds_now.flagsPack_now[t], nerv.TNN.FC.HAS_LABEL) > 0 then for i = 1, #self.dim_out do @@ -489,6 +510,9 @@ end --ref: the TNN_ref of a layer --t: the current time to propagate function TNN:backpropagate_dfs(ref, t, do_update) + if do_update == nil then + nerv.error("got a nil do_update") + end if self:out_of_feedrange(t) then return end |