From 317ff51cae8dcfaff26855c42ce99656b4d293b5 Mon Sep 17 00:00:00 2001 From: txh18 Date: Tue, 17 Nov 2015 13:20:43 +0800 Subject: added small opt: use mmatrix in lm_trainer and reader --- nerv/examples/lmptb/lm_trainer.lua | 17 +- nerv/examples/lmptb/lmptb/lmseqreader.lua | 28 +- nerv/examples/lmptb/m-tests/LOG-tnn-h400 | 10791 +++++++++++++++------------- nerv/examples/lmptb/rnn/tnn.lua | 16 +- 4 files changed, 5688 insertions(+), 5164 deletions(-) diff --git a/nerv/examples/lmptb/lm_trainer.lua b/nerv/examples/lmptb/lm_trainer.lua index 7dd70e2..62d8b50 100644 --- a/nerv/examples/lmptb/lm_trainer.lua +++ b/nerv/examples/lmptb/lm_trainer.lua @@ -15,17 +15,18 @@ function LMTrainer.lm_process_file(global_conf, fn, tnn, do_train) reader:open_file(fn) local result = nerv.LMResult(global_conf, global_conf.vocab) result:init("rnn") - + global_conf.timer:flush() tnn:flush_all() --caution: will also flush the inputs from the reader! local next_log_wcn = global_conf.log_w_num + local neto_bakm = global_conf.mmat_type(global_conf.batch_size, 1) --space backup matrix for network output while (1) do global_conf.timer:tic('most_out_loop_lmprocessfile') local r, feeds - + global_conf.timer:tic('tnn_beforeprocess') r, feeds = tnn:getfeed_from_reader(reader) if r == false then break @@ -39,6 +40,7 @@ function LMTrainer.lm_process_file(global_conf, fn, tnn, do_train) end end end + global_conf.timer:toc('tnn_beforeprocess') --[[ for j = 1, global_conf.chunk_size, 1 do @@ -56,15 +58,20 @@ function LMTrainer.lm_process_file(global_conf, fn, tnn, do_train) tnn:net_backpropagate(false) tnn:net_backpropagate(true) end - + + global_conf.timer:tic('tnn_afterprocess') for t = 1, global_conf.chunk_size, 1 do + tnn.outputs_m[t][1]:copy_toh(neto_bakm) for i = 1, global_conf.batch_size, 1 do if (feeds.labels_s[t][i] ~= global_conf.vocab.null_token) then - result:add("rnn", feeds.labels_s[t][i], math.exp(tnn.outputs_m[t][1][i - 1][0])) + --result:add("rnn", feeds.labels_s[t][i], math.exp(tnn.outputs_m[t][1][i - 1][0])) + result:add("rnn", feeds.labels_s[t][i], math.exp(neto_bakm[i - 1][0])) end end end - tnn:move_right_to_nextmb() + tnn:move_right_to_nextmb({0}) --only copy for time 0 + global_conf.timer:toc('tnn_afterprocess') + global_conf.timer:toc('most_out_loop_lmprocessfile') --print log diff --git a/nerv/examples/lmptb/lmptb/lmseqreader.lua b/nerv/examples/lmptb/lmptb/lmseqreader.lua index e0dcd95..cc805a4 100644 --- a/nerv/examples/lmptb/lmptb/lmseqreader.lua +++ b/nerv/examples/lmptb/lmptb/lmseqreader.lua @@ -30,6 +30,13 @@ function LMReader:open_file(fn) for i = 1, self.batch_size, 1 do self.streams[i] = {["store"] = {}, ["head"] = 1, ["tail"] = 0} end + + self.bak_inputs_m = {} --backup MMatrix for temporary storey(then copy to TNN CuMatrix) + for j = 1, self.chunk_size, 1 do + self.bak_inputs_m[j] = {} + self.bak_inputs_m[j][1] = self.gconf.mmat_type(self.batch_size, 1) + self.bak_inputs_m[j][2] = self.gconf.mmat_type(self.batch_size, self.vocab:size()) --since MMatrix does not yet have fill, this m[j][2] is not used + end end --id: int @@ -78,7 +85,7 @@ function LMReader:get_batch(feeds) local labels_s = feeds.labels_s for i = 1, self.chunk_size, 1 do inputs_s[i] = {} - labels_s[i] = {} + labels_s[i] = {} end local inputs_m = feeds.inputs_m --port 1 : word_id, port 2 : label @@ -86,20 +93,24 @@ function LMReader:get_batch(feeds) local flagsPack = feeds.flagsPack_now local got_new = false + for j = 1, self.chunk_size, 1 do + inputs_m[j][2]:fill(0) + end for i = 1, self.batch_size, 1 do local st = self.streams[i] for j = 1, self.chunk_size, 1 do flags[j][i] = 0 self:refresh_stream(i) - if (st.store[st.head] ~= nil) then + if st.store[st.head] ~= nil then inputs_s[j][i] = st.store[st.head] - inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head]).id - 1 + --inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head]).id - 1 + self.bak_inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head]).id - 1 else inputs_s[j][i] = self.vocab.null_token - inputs_m[j][1][i - 1][0] = 0 + --inputs_m[j][1][i - 1][0] = 0 + self.bak_inputs_m[j][1][i - 1][0] = 0 end - inputs_m[j][2][i - 1]:fill(0) - if (st.store[st.head + 1] ~= nil) then + if st.store[st.head + 1] ~= nil then labels_s[j][i] = st.store[st.head + 1] inputs_m[j][2][i - 1][self.vocab:get_word_str(st.store[st.head + 1]).id - 1] = 1 else @@ -116,12 +127,12 @@ function LMReader:get_batch(feeds) got_new = true st.store[st.head] = nil st.head = st.head + 1 - if (labels_s[j][i] == self.vocab.sen_end_token) then + if labels_s[j][i] == self.vocab.sen_end_token then flags[j][i] = bit.bor(flags[j][i], nerv.TNN.FC.SEQ_END) st.store[st.head] = nil --sentence end is passed st.head = st.head + 1 end - if (inputs_s[j][i] == self.vocab.sen_end_token) then + if inputs_s[j][i] == self.vocab.sen_end_token then flags[j][i] = bit.bor(flags[j][i], nerv.TNN.FC.SEQ_START) end end @@ -133,6 +144,7 @@ function LMReader:get_batch(feeds) for i = 1, self.batch_size, 1 do flagsPack[j] = bit.bor(flagsPack[j], flags[j][i]) end + inputs_m[j][1]:copy_fromh(self.bak_inputs_m[j][1]) end if (got_new == false) then diff --git a/nerv/examples/lmptb/m-tests/LOG-tnn-h400 b/nerv/examples/lmptb/m-tests/LOG-tnn-h400 index e562db6..b0209f6 100644 --- a/nerv/examples/lmptb/m-tests/LOG-tnn-h400 +++ b/nerv/examples/lmptb/m-tests/LOG-tnn-h400 @@ -1,12 +1,12 @@ Greetings -[SCHEDULER]: applying arg[2](start_iter=3)... +[SCHEDULER]: not user setting, all default... [SCHEDULER]: printing global_conf... test_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds train_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.train.txt.adds -param_random function: 0x414d5958 +param_random function: 0x4121a7c8 train_fn_shuf_bak /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf_bak decay_iter 16 -mmat_type table: 0x410c5058 +mmat_type table: 0x415bc3f0 vocab_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/vocab train_fn_shuf /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf param_fn /home/slhome/txh18/workspace/nerv/play/dagL_test/params @@ -23,42 +23,534 @@ lrate 1 momentum 0 wcost 1e-05 chunk_size 15 -cumat_type table: 0x4073aa48 +cumat_type table: 0x40f7d918 [SCHEDULER]: printing training scheduling options... lr_half false -start_iter 3 +start_iter -1 ppl_last 100000 [SCHEDULER]: printing training scheduling end. [SCHEDULER]: creating work_dir... [SCHEDULER]: building vocab... [LOG]LMVocab: Vocab building on file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/vocab... [LOG]LMVocab: Building finished, vocab size now is 10000. +[SCHEDULER]: preparing parameters... +[SCHEDULER]: first time, generating parameters... +===INITIAL VALIDATION=== +[SCHEDULER]: preparing parameters... +[SCHEDULER]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.0... +reading chunk 0 from 0 +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} + +reading chunk 1 from 46010296 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} + +reading chunk 2 from 47850982 +metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} + +reading chunk 3 from 93850772 +metadata: return {type="nerv.BiasParam",id="bp_h"} + +reading chunk 4 from 93855417 +metadata: return {type="nerv.BiasParam",id="bp_o"} + +reading chunk 5 from 93970392 +[SCHEDULER]: preparing parameters end. +[SCHEDULER]: preparing layers... +(00:04:37 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:04:37 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:04:37 2015-11-17)[nerv] info: create layer: combinerL1 +(00:04:37 2015-11-17)[nerv] info: create layer: outputL +(00:04:37 2015-11-17)[nerv] info: create layer: softmaxL +(00:04:37 2015-11-17)[nerv] info: create layer: selectL1 +[SCHEDULER]: preparing layers end. +[SCHEDULER]: Generate and initing TNN ... + selectL1 0 +selectL1 recurrentL1 0 +recurrentL1 sigmoidL1 0 +sigmoidL1 combinerL1 0 +combinerL1 recurrentL1 1 +combinerL1 outputL 0 +outputL softmaxL 0 + softmaxL 0 +softmaxL 0 +recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 +outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 +sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 +softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 +TNN initing storage selectL1 -> recurrentL1 +TNN initing storage recurrentL1 -> sigmoidL1 +TNN initing storage sigmoidL1 -> combinerL1 +TNN initing storage combinerL1 -> recurrentL1 +TNN initing storage combinerL1 -> outputL +TNN initing storage outputL -> softmaxL +[SCHEDULER]: Initing TNN end. +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER]: 40095 words processed Tue Nov 17 00:04:43 2015. + [SCHEDULER]: log prob per sample :-4.031438. + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74157 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67356 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER]: Displaying result: +[SCHEDULER]: LMResult status of rnn: +[SCHEDULER]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. + +[SCHEDULER ITER1 LR1]: preparing parameters... +[SCHEDULER ITER1 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.0... +reading chunk 0 from 0 +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} + +reading chunk 1 from 46010296 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} + +reading chunk 2 from 47850982 +metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} + +reading chunk 3 from 93850772 +metadata: return {type="nerv.BiasParam",id="bp_h"} + +reading chunk 4 from 93855417 +metadata: return {type="nerv.BiasParam",id="bp_o"} + +reading chunk 5 from 93970392 +[SCHEDULER ITER1 LR1]: preparing parameters end. +[SCHEDULER ITER1 LR1]: preparing layers... +(00:04:51 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:04:51 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:04:51 2015-11-17)[nerv] info: create layer: combinerL1 +(00:04:51 2015-11-17)[nerv] info: create layer: outputL +(00:04:51 2015-11-17)[nerv] info: create layer: softmaxL +(00:04:51 2015-11-17)[nerv] info: create layer: selectL1 +[SCHEDULER ITER1 LR1]: preparing layers end. +[SCHEDULER ITER1 LR1]: Generate and initing TNN ... + selectL1 0 +selectL1 recurrentL1 0 +recurrentL1 sigmoidL1 0 +sigmoidL1 combinerL1 0 +combinerL1 recurrentL1 1 +combinerL1 outputL 0 +outputL softmaxL 0 + softmaxL 0 +softmaxL 0 +recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 +outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 +sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 +softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 +TNN initing storage selectL1 -> recurrentL1 +TNN initing storage recurrentL1 -> sigmoidL1 +TNN initing storage sigmoidL1 -> combinerL1 +TNN initing storage combinerL1 -> recurrentL1 +TNN initing storage combinerL1 -> outputL +TNN initing storage outputL -> softmaxL +[SCHEDULER ITER1 LR1]: Initing TNN end. +===ITERATION 1 LR 1.000000=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER1 LR1]: 40099 words processed Tue Nov 17 00:05:05 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.302078. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.01607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92373 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42845 clock time +[SCHEDULER ITER1 LR1]: 80063 words processed Tue Nov 17 00:05:19 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.154478. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.98043 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.90939 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42106 clock time +[SCHEDULER ITER1 LR1]: 120068 words processed Tue Nov 17 00:05:33 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.071595. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.93187 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61576 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41444 clock time +[SCHEDULER ITER1 LR1]: 160017 words processed Tue Nov 17 00:05:47 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.003927. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.93631 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89976 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61828 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41460 clock time +[SCHEDULER ITER1 LR1]: 200138 words processed Tue Nov 17 00:06:01 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.966038. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.03957 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.65248 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43007 clock time +[SCHEDULER ITER1 LR1]: 240007 words processed Tue Nov 17 00:06:15 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.931091. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.90404 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.88924 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.60917 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.40914 clock time +[SCHEDULER ITER1 LR1]: 280135 words processed Tue Nov 17 00:06:29 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.902398. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.06335 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93462 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66248 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43464 clock time +[SCHEDULER ITER1 LR1]: 320080 words processed Tue Nov 17 00:06:43 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.877023. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.01798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.91726 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.65129 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42261 clock time +[SCHEDULER ITER1 LR1]: 360059 words processed Tue Nov 17 00:06:57 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.849770. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.92666 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89945 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61643 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41522 clock time +[SCHEDULER ITER1 LR1]: 400021 words processed Tue Nov 17 00:07:11 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.831115. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.97814 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.90838 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63725 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41697 clock time +[SCHEDULER ITER1 LR1]: 440102 words processed Tue Nov 17 00:07:25 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.811526. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.07885 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93784 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.67447 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43639 clock time +[SCHEDULER ITER1 LR1]: 480051 words processed Tue Nov 17 00:07:39 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.793057. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.92534 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89979 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41501 clock time +[SCHEDULER ITER1 LR1]: 520093 words processed Tue Nov 17 00:07:53 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.776523. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.97012 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.91708 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.62721 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42344 clock time +[SCHEDULER ITER1 LR1]: 560039 words processed Tue Nov 17 00:08:07 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.759064. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.96381 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.90525 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63379 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41726 clock time +[SCHEDULER ITER1 LR1]: 600112 words processed Tue Nov 17 00:08:21 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.747706. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.99693 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92212 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63992 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42553 clock time +[SCHEDULER ITER1 LR1]: 640076 words processed Tue Nov 17 00:08:35 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.735499. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.92614 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89935 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61681 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41585 clock time +[SCHEDULER ITER1 LR1]: 680026 words processed Tue Nov 17 00:08:49 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.725202. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.95927 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.90817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.62683 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42190 clock time +[SCHEDULER ITER1 LR1]: 720133 words processed Tue Nov 17 00:09:03 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.715340. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.01923 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64506 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43106 clock time +[SCHEDULER ITER1 LR1]: 760048 words processed Tue Nov 17 00:09:17 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.703347. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.00692 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92104 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42213 clock time +[SCHEDULER ITER1 LR1]: 800117 words processed Tue Nov 17 00:09:31 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.694935. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.10784 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94340 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.67631 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43240 clock time +[SCHEDULER ITER1 LR1]: 840116 words processed Tue Nov 17 00:09:45 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.684513. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.99321 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92163 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63629 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42577 clock time +[SCHEDULER ITER1 LR1]: 880037 words processed Tue Nov 17 00:09:59 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.676402. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.97565 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.91157 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63393 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42184 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER1 LR1]: Displaying result: +[SCHEDULER ITER1 LR1]: LMResult status of rnn: +[SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER1 LR1]: shuffling training file +===PEEK ON TEST 1=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER1 LR1]: 40087 words processed Tue Nov 17 00:10:12 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.512880. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.54269 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64145 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER1 LR1]: Displaying result: +[SCHEDULER ITER1 LR1]: LMResult status of rnn: +[SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +===VALIDATION 1=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER1 LR1]: 40095 words processed Tue Nov 17 00:10:25 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.556837. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.59126 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66055 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER1 LR1]: Displaying result: +[SCHEDULER ITER1 LR1]: LMResult status of rnn: +[SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER1 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.1... + +[SCHEDULER ITER2 LR1]: preparing parameters... +[SCHEDULER ITER2 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.1... +reading chunk 0 from 0 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} + +reading chunk 1 from 1911286 +metadata: return {type="nerv.BiasParam",id="bp_h"} + +reading chunk 2 from 1916142 +metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} + +reading chunk 3 from 47938924 +metadata: return {type="nerv.BiasParam",id="bp_o"} + +reading chunk 4 from 48055301 +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} + +reading chunk 5 from 94077793 +[SCHEDULER ITER2 LR1]: preparing parameters end. +[SCHEDULER ITER2 LR1]: preparing layers... +(00:10:36 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:10:36 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:10:36 2015-11-17)[nerv] info: create layer: combinerL1 +(00:10:36 2015-11-17)[nerv] info: create layer: outputL +(00:10:36 2015-11-17)[nerv] info: create layer: softmaxL +(00:10:36 2015-11-17)[nerv] info: create layer: selectL1 +[SCHEDULER ITER2 LR1]: preparing layers end. +[SCHEDULER ITER2 LR1]: Generate and initing TNN ... + selectL1 0 +selectL1 recurrentL1 0 +recurrentL1 sigmoidL1 0 +sigmoidL1 combinerL1 0 +combinerL1 recurrentL1 1 +combinerL1 outputL 0 +outputL softmaxL 0 + softmaxL 0 +softmaxL 0 +recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 +outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 +sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 +softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 +TNN initing storage selectL1 -> recurrentL1 +TNN initing storage recurrentL1 -> sigmoidL1 +TNN initing storage sigmoidL1 -> combinerL1 +TNN initing storage combinerL1 -> recurrentL1 +TNN initing storage combinerL1 -> outputL +TNN initing storage outputL -> softmaxL +[SCHEDULER ITER2 LR1]: Initing TNN end. +===ITERATION 2 LR 1.000000=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER2 LR1]: 40092 words processed Tue Nov 17 00:10:50 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.692655. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21693 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97564 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73263 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42816 clock time +[SCHEDULER ITER2 LR1]: 80099 words processed Tue Nov 17 00:11:04 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.630314. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95483 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72654 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42019 clock time +[SCHEDULER ITER2 LR1]: 120004 words processed Tue Nov 17 00:11:19 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.593422. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18646 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95653 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73055 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42199 clock time +[SCHEDULER ITER2 LR1]: 160114 words processed Tue Nov 17 00:11:34 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.576778. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97606 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74550 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43213 clock time +[SCHEDULER ITER2 LR1]: 200066 words processed Tue Nov 17 00:11:49 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.566877. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19386 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42575 clock time +[SCHEDULER ITER2 LR1]: 240045 words processed Tue Nov 17 00:12:04 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.557151. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21182 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96601 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74369 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42686 clock time +[SCHEDULER ITER2 LR1]: 280057 words processed Tue Nov 17 00:12:19 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.551346. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21951 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97537 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73797 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43301 clock time +[SCHEDULER ITER2 LR1]: 320106 words processed Tue Nov 17 00:12:34 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.543334. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22415 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97600 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74243 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43203 clock time +[SCHEDULER ITER2 LR1]: 360024 words processed Tue Nov 17 00:12:49 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.538019. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19260 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73014 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42517 clock time +[SCHEDULER ITER2 LR1]: 400089 words processed Tue Nov 17 00:13:04 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.534185. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22895 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97811 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74590 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43174 clock time +[SCHEDULER ITER2 LR1]: 440067 words processed Tue Nov 17 00:13:19 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.530238. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26819 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98086 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76123 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43585 clock time +[SCHEDULER ITER2 LR1]: 480051 words processed Tue Nov 17 00:13:33 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.526657. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18099 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96648 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72488 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42495 clock time +[SCHEDULER ITER2 LR1]: 520140 words processed Tue Nov 17 00:13:49 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.523109. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34003 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00399 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78469 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44669 clock time +[SCHEDULER ITER2 LR1]: 560132 words processed Tue Nov 17 00:14:03 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.519788. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15648 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95715 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72157 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42384 clock time +[SCHEDULER ITER2 LR1]: 600118 words processed Tue Nov 17 00:14:18 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.519174. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20357 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96149 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74400 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42778 clock time +[SCHEDULER ITER2 LR1]: 640090 words processed Tue Nov 17 00:14:32 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.515980. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96122 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42133 clock time +[SCHEDULER ITER2 LR1]: 680075 words processed Tue Nov 17 00:14:47 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.513608. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21724 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97093 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74363 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43023 clock time +[SCHEDULER ITER2 LR1]: 720043 words processed Tue Nov 17 00:15:01 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.511130. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16473 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95315 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73050 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42120 clock time +[SCHEDULER ITER2 LR1]: 760012 words processed Tue Nov 17 00:15:16 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.508404. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18821 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96161 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73252 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42577 clock time +[SCHEDULER ITER2 LR1]: 800113 words processed Tue Nov 17 00:15:31 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.505336. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18815 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72943 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42949 clock time +[SCHEDULER ITER2 LR1]: 840089 words processed Tue Nov 17 00:15:45 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.501920. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14105 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95144 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71526 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42053 clock time +[SCHEDULER ITER2 LR1]: 880052 words processed Tue Nov 17 00:16:00 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.499207. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21738 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97053 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74261 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43068 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER2 LR1]: Displaying result: +[SCHEDULER ITER2 LR1]: LMResult status of rnn: +[SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER2 LR1]: shuffling training file +===PEEK ON TEST 2=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER2 LR1]: 40087 words processed Tue Nov 17 00:16:11 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.397402. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67220 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74227 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER2 LR1]: Displaying result: +[SCHEDULER ITER2 LR1]: LMResult status of rnn: +[SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +===VALIDATION 2=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER2 LR1]: 40095 words processed Tue Nov 17 00:16:24 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.442373. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.64900 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72733 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER2 LR1]: Displaying result: +[SCHEDULER ITER2 LR1]: LMResult status of rnn: +[SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER2 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.2... + [SCHEDULER ITER3 LR1]: preparing parameters... [SCHEDULER ITER3 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.2... reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46029692 +reading chunk 1 from 46026912 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46034550 +reading chunk 2 from 46031770 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92091219 +reading chunk 3 from 92077295 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92207421 +reading chunk 4 from 92193586 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94117132 +reading chunk 5 from 94104045 [SCHEDULER ITER3 LR1]: preparing parameters end. [SCHEDULER ITER3 LR1]: preparing layers... -(19:23:36 2015-11-16)[nerv] info: create layer: recurrentL1 -(19:23:36 2015-11-16)[nerv] info: create layer: sigmoidL1 -(19:23:36 2015-11-16)[nerv] info: create layer: combinerL1 -(19:23:36 2015-11-16)[nerv] info: create layer: outputL -(19:23:36 2015-11-16)[nerv] info: create layer: softmaxL -(19:23:36 2015-11-16)[nerv] info: create layer: selectL1 +(00:16:35 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:16:35 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:16:35 2015-11-17)[nerv] info: create layer: combinerL1 +(00:16:35 2015-11-17)[nerv] info: create layer: outputL +(00:16:35 2015-11-17)[nerv] info: create layer: softmaxL +(00:16:35 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER3 LR1]: preparing layers end. [SCHEDULER ITER3 LR1]: Generate and initing TNN ... selectL1 0 @@ -86,164 +578,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 3 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER3 LR1]: 40099 words processed Mon Nov 16 19:23:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.401654. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25687 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04683 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43521 clock time -[SCHEDULER ITER3 LR1]: 80063 words processed Mon Nov 16 19:24:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.374433. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19857 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02717 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71469 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42511 clock time -[SCHEDULER ITER3 LR1]: 120068 words processed Mon Nov 16 19:24:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.378173. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31757 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05634 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76081 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43819 clock time -[SCHEDULER ITER3 LR1]: 160017 words processed Mon Nov 16 19:24:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.366419. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21593 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02930 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72744 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42766 clock time -[SCHEDULER ITER3 LR1]: 200138 words processed Mon Nov 16 19:24:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.375391. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06194 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44354 clock time -[SCHEDULER ITER3 LR1]: 240007 words processed Mon Nov 16 19:25:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.374863. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15444 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01010 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71058 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41652 clock time -[SCHEDULER ITER3 LR1]: 280135 words processed Mon Nov 16 19:25:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.371663. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32493 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06209 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77125 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44028 clock time -[SCHEDULER ITER3 LR1]: 320080 words processed Mon Nov 16 19:25:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.372572. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22077 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73036 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42811 clock time -[SCHEDULER ITER3 LR1]: 360059 words processed Mon Nov 16 19:25:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.367750. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30888 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05146 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77331 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43544 clock time -[SCHEDULER ITER3 LR1]: 400021 words processed Mon Nov 16 19:26:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.370284. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19708 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02923 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72020 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42434 clock time -[SCHEDULER ITER3 LR1]: 440102 words processed Mon Nov 16 19:26:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.366085. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22346 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04211 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72356 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43221 clock time -[SCHEDULER ITER3 LR1]: 480051 words processed Mon Nov 16 19:26:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.362942. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21148 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02936 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73411 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42472 clock time -[SCHEDULER ITER3 LR1]: 520093 words processed Mon Nov 16 19:26:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.359600. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26407 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05013 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43525 clock time -[SCHEDULER ITER3 LR1]: 560039 words processed Mon Nov 16 19:27:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.352747. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28053 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43241 clock time -[SCHEDULER ITER3 LR1]: 600112 words processed Mon Nov 16 19:27:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.354185. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28527 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05332 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75468 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43716 clock time -[SCHEDULER ITER3 LR1]: 640076 words processed Mon Nov 16 19:27:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.351938. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17509 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02320 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71112 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42335 clock time -[SCHEDULER ITER3 LR1]: 680026 words processed Mon Nov 16 19:27:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.352402. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19184 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02506 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72222 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42376 clock time -[SCHEDULER ITER3 LR1]: 720133 words processed Mon Nov 16 19:28:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.351590. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22742 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04343 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72700 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43243 clock time -[SCHEDULER ITER3 LR1]: 760048 words processed Mon Nov 16 19:28:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.347382. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19520 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02816 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72164 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42459 clock time -[SCHEDULER ITER3 LR1]: 800117 words processed Mon Nov 16 19:28:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.346849. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33851 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06692 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77792 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43937 clock time -[SCHEDULER ITER3 LR1]: 840116 words processed Mon Nov 16 19:28:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.342472. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32912 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76887 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44276 clock time -[SCHEDULER ITER3 LR1]: 880037 words processed Mon Nov 16 19:29:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.341243. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22162 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03318 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72779 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43004 clock time +[SCHEDULER ITER3 LR1]: 40092 words processed Tue Nov 17 00:16:49 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.451944. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24393 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97312 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75603 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43213 clock time +[SCHEDULER ITER3 LR1]: 80099 words processed Tue Nov 17 00:17:03 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.440249. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16343 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94441 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73597 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41776 clock time +[SCHEDULER ITER3 LR1]: 120004 words processed Tue Nov 17 00:17:17 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.429951. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15249 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94180 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72770 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41684 clock time +[SCHEDULER ITER3 LR1]: 160114 words processed Tue Nov 17 00:17:32 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.428219. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19216 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95969 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73701 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42746 clock time +[SCHEDULER ITER3 LR1]: 200066 words processed Tue Nov 17 00:17:47 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.427816. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19681 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96944 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74314 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41611 clock time +[SCHEDULER ITER3 LR1]: 240045 words processed Tue Nov 17 00:18:02 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.421879. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20117 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95415 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75101 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42315 clock time +[SCHEDULER ITER3 LR1]: 280057 words processed Tue Nov 17 00:18:17 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.418622. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26805 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97981 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76757 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43297 clock time +[SCHEDULER ITER3 LR1]: 320106 words processed Tue Nov 17 00:18:32 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.413292. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22311 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96642 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75295 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42966 clock time +[SCHEDULER ITER3 LR1]: 360024 words processed Tue Nov 17 00:18:47 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.410893. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21722 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95975 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75310 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42653 clock time +[SCHEDULER ITER3 LR1]: 400089 words processed Tue Nov 17 00:19:02 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.409763. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33080 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98658 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79905 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44089 clock time +[SCHEDULER ITER3 LR1]: 440067 words processed Tue Nov 17 00:19:17 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.408198. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19522 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95467 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75045 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42259 clock time +[SCHEDULER ITER3 LR1]: 480051 words processed Tue Nov 17 00:19:31 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.406487. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12384 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94097 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71649 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41720 clock time +[SCHEDULER ITER3 LR1]: 520140 words processed Tue Nov 17 00:19:46 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.404808. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19942 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95834 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75314 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42578 clock time +[SCHEDULER ITER3 LR1]: 560132 words processed Tue Nov 17 00:20:01 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.403215. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20221 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95699 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74750 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42386 clock time +[SCHEDULER ITER3 LR1]: 600118 words processed Tue Nov 17 00:20:16 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.404832. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29962 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97894 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78289 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43809 clock time +[SCHEDULER ITER3 LR1]: 640090 words processed Tue Nov 17 00:20:31 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.402722. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21695 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95831 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75755 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42378 clock time +[SCHEDULER ITER3 LR1]: 680075 words processed Tue Nov 17 00:20:45 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.401527. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15485 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94411 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73283 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41815 clock time +[SCHEDULER ITER3 LR1]: 720043 words processed Tue Nov 17 00:20:59 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.400318. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94281 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71709 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41870 clock time +[SCHEDULER ITER3 LR1]: 760012 words processed Tue Nov 17 00:21:14 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.398883. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19500 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95331 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75083 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42308 clock time +[SCHEDULER ITER3 LR1]: 800113 words processed Tue Nov 17 00:21:29 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.397416. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96361 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74747 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42731 clock time +[SCHEDULER ITER3 LR1]: 840089 words processed Tue Nov 17 00:21:43 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.395775. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12880 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94176 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71968 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41782 clock time +[SCHEDULER ITER3 LR1]: 880052 words processed Tue Nov 17 00:21:58 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.394509. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21553 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95974 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42560 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER3 LR1]: Displaying result: -[SCHEDULER ITER3 LR1]: LMResult status of rnn: +[SCHEDULER ITER3 LR1]: LMResult status of rnn: [SCHEDULER ITER3 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER3 LR1]: shuffling training file ===PEEK ON TEST 3=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER3 LR1]: 40087 words processed Mon Nov 16 19:29:18 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.354254. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78797 clock time +[SCHEDULER ITER3 LR1]: 40087 words processed Tue Nov 17 00:22:09 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.340680. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68311 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75306 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER3 LR1]: Displaying result: -[SCHEDULER ITER3 LR1]: LMResult status of rnn: +[SCHEDULER ITER3 LR1]: LMResult status of rnn: [SCHEDULER ITER3 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 3=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER3 LR1]: 40095 words processed Mon Nov 16 19:29:31 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.410570. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.71311 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77428 clock time +[SCHEDULER ITER3 LR1]: 40095 words processed Tue Nov 17 00:22:22 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.390081. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67167 clock time + [gl