diff options
| -rw-r--r-- | nerv/examples/lmptb/lm_trainer.lua | 17 | ||||
| -rw-r--r-- | nerv/examples/lmptb/lmptb/lmseqreader.lua | 28 | ||||
| -rw-r--r-- | nerv/examples/lmptb/m-tests/LOG-tnn-h400 | 10791 | ||||
| -rw-r--r-- | nerv/examples/lmptb/rnn/tnn.lua | 16 |
4 files changed, 5688 insertions, 5164 deletions
diff --git a/nerv/examples/lmptb/lm_trainer.lua b/nerv/examples/lmptb/lm_trainer.lua index 7dd70e2..62d8b50 100644 --- a/nerv/examples/lmptb/lm_trainer.lua +++ b/nerv/examples/lmptb/lm_trainer.lua @@ -15,17 +15,18 @@ function LMTrainer.lm_process_file(global_conf, fn, tnn, do_train) reader:open_file(fn) local result = nerv.LMResult(global_conf, global_conf.vocab) result:init("rnn") - + global_conf.timer:flush() tnn:flush_all() --caution: will also flush the inputs from the reader! local next_log_wcn = global_conf.log_w_num + local neto_bakm = global_conf.mmat_type(global_conf.batch_size, 1) --space backup matrix for network output while (1) do global_conf.timer:tic('most_out_loop_lmprocessfile') local r, feeds - + global_conf.timer:tic('tnn_beforeprocess') r, feeds = tnn:getfeed_from_reader(reader) if r == false then break @@ -39,6 +40,7 @@ function LMTrainer.lm_process_file(global_conf, fn, tnn, do_train) end end end + global_conf.timer:toc('tnn_beforeprocess') --[[ for j = 1, global_conf.chunk_size, 1 do @@ -56,15 +58,20 @@ function LMTrainer.lm_process_file(global_conf, fn, tnn, do_train) tnn:net_backpropagate(false) tnn:net_backpropagate(true) end - + + global_conf.timer:tic('tnn_afterprocess') for t = 1, global_conf.chunk_size, 1 do + tnn.outputs_m[t][1]:copy_toh(neto_bakm) for i = 1, global_conf.batch_size, 1 do if (feeds.labels_s[t][i] ~= global_conf.vocab.null_token) then - result:add("rnn", feeds.labels_s[t][i], math.exp(tnn.outputs_m[t][1][i - 1][0])) + --result:add("rnn", feeds.labels_s[t][i], math.exp(tnn.outputs_m[t][1][i - 1][0])) + result:add("rnn", feeds.labels_s[t][i], math.exp(neto_bakm[i - 1][0])) end end end - tnn:move_right_to_nextmb() + tnn:move_right_to_nextmb({0}) --only copy for time 0 + global_conf.timer:toc('tnn_afterprocess') + global_conf.timer:toc('most_out_loop_lmprocessfile') --print log diff --git a/nerv/examples/lmptb/lmptb/lmseqreader.lua b/nerv/examples/lmptb/lmptb/lmseqreader.lua index e0dcd95..cc805a4 100644 --- a/nerv/examples/lmptb/lmptb/lmseqreader.lua +++ b/nerv/examples/lmptb/lmptb/lmseqreader.lua @@ -30,6 +30,13 @@ function LMReader:open_file(fn) for i = 1, self.batch_size, 1 do self.streams[i] = {["store"] = {}, ["head"] = 1, ["tail"] = 0} end + + self.bak_inputs_m = {} --backup MMatrix for temporary storey(then copy to TNN CuMatrix) + for j = 1, self.chunk_size, 1 do + self.bak_inputs_m[j] = {} + self.bak_inputs_m[j][1] = self.gconf.mmat_type(self.batch_size, 1) + self.bak_inputs_m[j][2] = self.gconf.mmat_type(self.batch_size, self.vocab:size()) --since MMatrix does not yet have fill, this m[j][2] is not used + end end --id: int @@ -78,7 +85,7 @@ function LMReader:get_batch(feeds) local labels_s = feeds.labels_s for i = 1, self.chunk_size, 1 do inputs_s[i] = {} - labels_s[i] = {} + labels_s[i] = {} end local inputs_m = feeds.inputs_m --port 1 : word_id, port 2 : label @@ -86,20 +93,24 @@ function LMReader:get_batch(feeds) local flagsPack = feeds.flagsPack_now local got_new = false + for j = 1, self.chunk_size, 1 do + inputs_m[j][2]:fill(0) + end for i = 1, self.batch_size, 1 do local st = self.streams[i] for j = 1, self.chunk_size, 1 do flags[j][i] = 0 self:refresh_stream(i) - if (st.store[st.head] ~= nil) then + if st.store[st.head] ~= nil then inputs_s[j][i] = st.store[st.head] - inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head]).id - 1 + --inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head]).id - 1 + self.bak_inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head]).id - 1 else inputs_s[j][i] = self.vocab.null_token - inputs_m[j][1][i - 1][0] = 0 + --inputs_m[j][1][i - 1][0] = 0 + self.bak_inputs_m[j][1][i - 1][0] = 0 end - inputs_m[j][2][i - 1]:fill(0) - if (st.store[st.head + 1] ~= nil) then + if st.store[st.head + 1] ~= nil then labels_s[j][i] = st.store[st.head + 1] inputs_m[j][2][i - 1][self.vocab:get_word_str(st.store[st.head + 1]).id - 1] = 1 else @@ -116,12 +127,12 @@ function LMReader:get_batch(feeds) got_new = true st.store[st.head] = nil st.head = st.head + 1 - if (labels_s[j][i] == self.vocab.sen_end_token) then + if labels_s[j][i] == self.vocab.sen_end_token then flags[j][i] = bit.bor(flags[j][i], nerv.TNN.FC.SEQ_END) st.store[st.head] = nil --sentence end is passed st.head = st.head + 1 end - if (inputs_s[j][i] == self.vocab.sen_end_token) then + if inputs_s[j][i] == self.vocab.sen_end_token then flags[j][i] = bit.bor(flags[j][i], nerv.TNN.FC.SEQ_START) end end @@ -133,6 +144,7 @@ function LMReader:get_batch(feeds) for i = 1, self.batch_size, 1 do flagsPack[j] = bit.bor(flagsPack[j], flags[j][i]) end + inputs_m[j][1]:copy_fromh(self.bak_inputs_m[j][1]) end if (got_new == false) then diff --git a/nerv/examples/lmptb/m-tests/LOG-tnn-h400 b/nerv/examples/lmptb/m-tests/LOG-tnn-h400 index e562db6..b0209f6 100644 --- a/nerv/examples/lmptb/m-tests/LOG-tnn-h400 +++ b/nerv/examples/lmptb/m-tests/LOG-tnn-h400 @@ -1,12 +1,12 @@ Greetings -[SCHEDULER]: applying arg[2](start_iter=3)... +[SCHEDULER]: not user setting, all default... [SCHEDULER]: printing global_conf... test_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds train_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.train.txt.adds -param_random function: 0x414d5958 +param_random function: 0x4121a7c8 train_fn_shuf_bak /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf_bak decay_iter 16 -mmat_type table: 0x410c5058 +mmat_type table: 0x415bc3f0 vocab_fn /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/vocab train_fn_shuf /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf param_fn /home/slhome/txh18/workspace/nerv/play/dagL_test/params @@ -23,42 +23,534 @@ lrate 1 momentum 0 wcost 1e-05 chunk_size 15 -cumat_type table: 0x4073aa48 +cumat_type table: 0x40f7d918 [SCHEDULER]: printing training scheduling options... lr_half false -start_iter 3 +start_iter -1 ppl_last 100000 [SCHEDULER]: printing training scheduling end. [SCHEDULER]: creating work_dir... [SCHEDULER]: building vocab... [LOG]LMVocab: Vocab building on file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/vocab... [LOG]LMVocab: Building finished, vocab size now is 10000. +[SCHEDULER]: preparing parameters... +[SCHEDULER]: first time, generating parameters... +===INITIAL VALIDATION=== +[SCHEDULER]: preparing parameters... +[SCHEDULER]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.0... +reading chunk 0 from 0 +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} + +reading chunk 1 from 46010296 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} + +reading chunk 2 from 47850982 +metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} + +reading chunk 3 from 93850772 +metadata: return {type="nerv.BiasParam",id="bp_h"} + +reading chunk 4 from 93855417 +metadata: return {type="nerv.BiasParam",id="bp_o"} + +reading chunk 5 from 93970392 +[SCHEDULER]: preparing parameters end. +[SCHEDULER]: preparing layers... +(00:04:37 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:04:37 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:04:37 2015-11-17)[nerv] info: create layer: combinerL1 +(00:04:37 2015-11-17)[nerv] info: create layer: outputL +(00:04:37 2015-11-17)[nerv] info: create layer: softmaxL +(00:04:37 2015-11-17)[nerv] info: create layer: selectL1 +[SCHEDULER]: preparing layers end. +[SCHEDULER]: Generate and initing TNN ... +<input> selectL1 0 +selectL1 recurrentL1 0 +recurrentL1 sigmoidL1 0 +sigmoidL1 combinerL1 0 +combinerL1 recurrentL1 1 +combinerL1 outputL 0 +outputL softmaxL 0 +<input> softmaxL 0 +softmaxL <output> 0 +recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 +outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 +sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 +softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 +TNN initing storage selectL1 -> recurrentL1 +TNN initing storage recurrentL1 -> sigmoidL1 +TNN initing storage sigmoidL1 -> combinerL1 +TNN initing storage combinerL1 -> recurrentL1 +TNN initing storage combinerL1 -> outputL +TNN initing storage outputL -> softmaxL +[SCHEDULER]: Initing TNN end. +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER]: 40095 words processed Tue Nov 17 00:04:43 2015. + [SCHEDULER]: log prob per sample :-4.031438. + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74157 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67356 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER]: Displaying result: +[SCHEDULER]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 11218.277022182> <PPL_OOV 10730.852429653> <LOGP -297299.58025201> +[SCHEDULER]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. + +[SCHEDULER ITER1 LR1]: preparing parameters... +[SCHEDULER ITER1 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.0... +reading chunk 0 from 0 +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} + +reading chunk 1 from 46010296 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} + +reading chunk 2 from 47850982 +metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} + +reading chunk 3 from 93850772 +metadata: return {type="nerv.BiasParam",id="bp_h"} + +reading chunk 4 from 93855417 +metadata: return {type="nerv.BiasParam",id="bp_o"} + +reading chunk 5 from 93970392 +[SCHEDULER ITER1 LR1]: preparing parameters end. +[SCHEDULER ITER1 LR1]: preparing layers... +(00:04:51 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:04:51 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:04:51 2015-11-17)[nerv] info: create layer: combinerL1 +(00:04:51 2015-11-17)[nerv] info: create layer: outputL +(00:04:51 2015-11-17)[nerv] info: create layer: softmaxL +(00:04:51 2015-11-17)[nerv] info: create layer: selectL1 +[SCHEDULER ITER1 LR1]: preparing layers end. +[SCHEDULER ITER1 LR1]: Generate and initing TNN ... +<input> selectL1 0 +selectL1 recurrentL1 0 +recurrentL1 sigmoidL1 0 +sigmoidL1 combinerL1 0 +combinerL1 recurrentL1 1 +combinerL1 outputL 0 +outputL softmaxL 0 +<input> softmaxL 0 +softmaxL <output> 0 +recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 +outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 +sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 +softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 +TNN initing storage selectL1 -> recurrentL1 +TNN initing storage recurrentL1 -> sigmoidL1 +TNN initing storage sigmoidL1 -> combinerL1 +TNN initing storage combinerL1 -> recurrentL1 +TNN initing storage combinerL1 -> outputL +TNN initing storage outputL -> softmaxL +[SCHEDULER ITER1 LR1]: Initing TNN end. +===ITERATION 1 LR 1.000000=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER1 LR1]: 40099 words processed Tue Nov 17 00:05:05 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.302078. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.01607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92373 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42845 clock time +[SCHEDULER ITER1 LR1]: 80063 words processed Tue Nov 17 00:05:19 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.154478. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.98043 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.90939 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42106 clock time +[SCHEDULER ITER1 LR1]: 120068 words processed Tue Nov 17 00:05:33 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.071595. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.93187 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61576 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41444 clock time +[SCHEDULER ITER1 LR1]: 160017 words processed Tue Nov 17 00:05:47 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-3.003927. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.93631 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89976 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61828 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41460 clock time +[SCHEDULER ITER1 LR1]: 200138 words processed Tue Nov 17 00:06:01 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.966038. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.03957 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.65248 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43007 clock time +[SCHEDULER ITER1 LR1]: 240007 words processed Tue Nov 17 00:06:15 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.931091. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.90404 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.88924 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.60917 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.40914 clock time +[SCHEDULER ITER1 LR1]: 280135 words processed Tue Nov 17 00:06:29 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.902398. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.06335 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93462 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66248 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43464 clock time +[SCHEDULER ITER1 LR1]: 320080 words processed Tue Nov 17 00:06:43 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.877023. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.01798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.91726 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.65129 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42261 clock time +[SCHEDULER ITER1 LR1]: 360059 words processed Tue Nov 17 00:06:57 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.849770. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.92666 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89945 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61643 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41522 clock time +[SCHEDULER ITER1 LR1]: 400021 words processed Tue Nov 17 00:07:11 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.831115. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.97814 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.90838 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63725 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41697 clock time +[SCHEDULER ITER1 LR1]: 440102 words processed Tue Nov 17 00:07:25 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.811526. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.07885 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93784 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.67447 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43639 clock time +[SCHEDULER ITER1 LR1]: 480051 words processed Tue Nov 17 00:07:39 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.793057. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.92534 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89979 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41501 clock time +[SCHEDULER ITER1 LR1]: 520093 words processed Tue Nov 17 00:07:53 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.776523. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.97012 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.91708 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.62721 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42344 clock time +[SCHEDULER ITER1 LR1]: 560039 words processed Tue Nov 17 00:08:07 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.759064. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.96381 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.90525 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63379 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41726 clock time +[SCHEDULER ITER1 LR1]: 600112 words processed Tue Nov 17 00:08:21 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.747706. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.99693 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92212 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63992 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42553 clock time +[SCHEDULER ITER1 LR1]: 640076 words processed Tue Nov 17 00:08:35 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.735499. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.92614 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.89935 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.61681 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41585 clock time +[SCHEDULER ITER1 LR1]: 680026 words processed Tue Nov 17 00:08:49 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.725202. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.95927 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.90817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.62683 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42190 clock time +[SCHEDULER ITER1 LR1]: 720133 words processed Tue Nov 17 00:09:03 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.715340. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.01923 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64506 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43106 clock time +[SCHEDULER ITER1 LR1]: 760048 words processed Tue Nov 17 00:09:17 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.703347. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.00692 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92104 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42213 clock time +[SCHEDULER ITER1 LR1]: 800117 words processed Tue Nov 17 00:09:31 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.694935. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.10784 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94340 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.67631 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43240 clock time +[SCHEDULER ITER1 LR1]: 840116 words processed Tue Nov 17 00:09:45 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.684513. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.99321 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92163 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63629 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42577 clock time +[SCHEDULER ITER1 LR1]: 880037 words processed Tue Nov 17 00:09:59 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.676402. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.97565 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.91157 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63393 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42184 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER1 LR1]: Displaying result: +[SCHEDULER ITER1 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 537.35706233668> <PPL_OOV 472.88518680998> <LOGP -2486423.4856078> +[SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER1 LR1]: shuffling training file +===PEEK ON TEST 1=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER1 LR1]: 40087 words processed Tue Nov 17 00:10:12 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.512880. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.54269 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64145 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER1 LR1]: Displaying result: +[SCHEDULER ITER1 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 376.36255129831> <PPL_OOV 331.0604978245> <LOGP -207715.96399617> +[SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +===VALIDATION 1=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER1 LR1]: 40095 words processed Tue Nov 17 00:10:25 2015. + [SCHEDULER ITER1 LR1]: log prob per sample :-2.556837. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.59126 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66055 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER1 LR1]: Displaying result: +[SCHEDULER ITER1 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 377.62321793049> <PPL_OOV 340.95708790642> <LOGP -186811.93157611> +[SCHEDULER ITER1 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER1 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.1... + +[SCHEDULER ITER2 LR1]: preparing parameters... +[SCHEDULER ITER2 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.1... +reading chunk 0 from 0 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} + +reading chunk 1 from 1911286 +metadata: return {type="nerv.BiasParam",id="bp_h"} + +reading chunk 2 from 1916142 +metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} + +reading chunk 3 from 47938924 +metadata: return {type="nerv.BiasParam",id="bp_o"} + +reading chunk 4 from 48055301 +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} + +reading chunk 5 from 94077793 +[SCHEDULER ITER2 LR1]: preparing parameters end. +[SCHEDULER ITER2 LR1]: preparing layers... +(00:10:36 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:10:36 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:10:36 2015-11-17)[nerv] info: create layer: combinerL1 +(00:10:36 2015-11-17)[nerv] info: create layer: outputL +(00:10:36 2015-11-17)[nerv] info: create layer: softmaxL +(00:10:36 2015-11-17)[nerv] info: create layer: selectL1 +[SCHEDULER ITER2 LR1]: preparing layers end. +[SCHEDULER ITER2 LR1]: Generate and initing TNN ... +<input> selectL1 0 +selectL1 recurrentL1 0 +recurrentL1 sigmoidL1 0 +sigmoidL1 combinerL1 0 +combinerL1 recurrentL1 1 +combinerL1 outputL 0 +outputL softmaxL 0 +<input> softmaxL 0 +softmaxL <output> 0 +recurrentL1 #dim_in: 2 #dim_out: 1 #i_conns_p: 2 #o_conns_p 1 +outputL #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +combinerL1 #dim_in: 1 #dim_out: 2 #i_conns_p: 1 #o_conns_p 2 +sigmoidL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 1 #o_conns_p 1 +selectL1 #dim_in: 1 #dim_out: 1 #i_conns_p: 0 #o_conns_p 1 +softmaxL #dim_in: 2 #dim_out: 1 #i_conns_p: 1 #o_conns_p 0 +TNN initing storage selectL1 -> recurrentL1 +TNN initing storage recurrentL1 -> sigmoidL1 +TNN initing storage sigmoidL1 -> combinerL1 +TNN initing storage combinerL1 -> recurrentL1 +TNN initing storage combinerL1 -> outputL +TNN initing storage outputL -> softmaxL +[SCHEDULER ITER2 LR1]: Initing TNN end. +===ITERATION 2 LR 1.000000=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER2 LR1]: 40092 words processed Tue Nov 17 00:10:50 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.692655. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21693 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97564 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73263 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42816 clock time +[SCHEDULER ITER2 LR1]: 80099 words processed Tue Nov 17 00:11:04 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.630314. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95483 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72654 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42019 clock time +[SCHEDULER ITER2 LR1]: 120004 words processed Tue Nov 17 00:11:19 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.593422. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18646 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95653 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73055 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42199 clock time +[SCHEDULER ITER2 LR1]: 160114 words processed Tue Nov 17 00:11:34 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.576778. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97606 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74550 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43213 clock time +[SCHEDULER ITER2 LR1]: 200066 words processed Tue Nov 17 00:11:49 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.566877. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19386 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42575 clock time +[SCHEDULER ITER2 LR1]: 240045 words processed Tue Nov 17 00:12:04 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.557151. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21182 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96601 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74369 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42686 clock time +[SCHEDULER ITER2 LR1]: 280057 words processed Tue Nov 17 00:12:19 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.551346. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21951 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97537 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73797 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43301 clock time +[SCHEDULER ITER2 LR1]: 320106 words processed Tue Nov 17 00:12:34 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.543334. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22415 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97600 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74243 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43203 clock time +[SCHEDULER ITER2 LR1]: 360024 words processed Tue Nov 17 00:12:49 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.538019. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19260 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73014 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42517 clock time +[SCHEDULER ITER2 LR1]: 400089 words processed Tue Nov 17 00:13:04 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.534185. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22895 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97811 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74590 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43174 clock time +[SCHEDULER ITER2 LR1]: 440067 words processed Tue Nov 17 00:13:19 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.530238. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26819 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98086 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76123 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43585 clock time +[SCHEDULER ITER2 LR1]: 480051 words processed Tue Nov 17 00:13:33 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.526657. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18099 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96648 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72488 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42495 clock time +[SCHEDULER ITER2 LR1]: 520140 words processed Tue Nov 17 00:13:49 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.523109. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34003 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00399 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78469 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44669 clock time +[SCHEDULER ITER2 LR1]: 560132 words processed Tue Nov 17 00:14:03 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.519788. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15648 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95715 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72157 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42384 clock time +[SCHEDULER ITER2 LR1]: 600118 words processed Tue Nov 17 00:14:18 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.519174. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20357 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96149 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74400 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42778 clock time +[SCHEDULER ITER2 LR1]: 640090 words processed Tue Nov 17 00:14:32 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.515980. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96122 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42133 clock time +[SCHEDULER ITER2 LR1]: 680075 words processed Tue Nov 17 00:14:47 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.513608. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21724 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97093 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74363 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43023 clock time +[SCHEDULER ITER2 LR1]: 720043 words processed Tue Nov 17 00:15:01 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.511130. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16473 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95315 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73050 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42120 clock time +[SCHEDULER ITER2 LR1]: 760012 words processed Tue Nov 17 00:15:16 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.508404. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18821 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96161 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73252 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42577 clock time +[SCHEDULER ITER2 LR1]: 800113 words processed Tue Nov 17 00:15:31 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.505336. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18815 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72943 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42949 clock time +[SCHEDULER ITER2 LR1]: 840089 words processed Tue Nov 17 00:15:45 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.501920. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14105 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95144 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71526 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42053 clock time +[SCHEDULER ITER2 LR1]: 880052 words processed Tue Nov 17 00:16:00 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.499207. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21738 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97053 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74261 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43068 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER2 LR1]: Displaying result: +[SCHEDULER ITER2 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 363.1890880336> <PPL_OOV 315.41499640977> <LOGP -2322933.5337178> +[SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. +[SCHEDULER ITER2 LR1]: shuffling training file +===PEEK ON TEST 2=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER2 LR1]: 40087 words processed Tue Nov 17 00:16:11 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.397402. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67220 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74227 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER2 LR1]: Displaying result: +[SCHEDULER ITER2 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 291.66785431418> <PPL_OOV 252.20424298756> <LOGP -197976.44940075> +[SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. +===VALIDATION 2=== +[LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... +batch_size: 10 chunk_size 15 +[SCHEDULER ITER2 LR1]: 40095 words processed Tue Nov 17 00:16:24 2015. + [SCHEDULER ITER2 LR1]: log prob per sample :-2.442373. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.64900 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72733 clock time +[LOG]LMSeqReader: file expires, closing. +[SCHEDULER ITER2 LR1]: Displaying result: +[SCHEDULER ITER2 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 295.99484209845> <PPL_OOV 263.05773346545> <LOGP -178502.96720832> +[SCHEDULER ITER2 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. +[SCHEDULER ITER2 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.2... + [SCHEDULER ITER3 LR1]: preparing parameters... [SCHEDULER ITER3 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.2... reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46029692 +reading chunk 1 from 46026912 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46034550 +reading chunk 2 from 46031770 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92091219 +reading chunk 3 from 92077295 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92207421 +reading chunk 4 from 92193586 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94117132 +reading chunk 5 from 94104045 [SCHEDULER ITER3 LR1]: preparing parameters end. [SCHEDULER ITER3 LR1]: preparing layers... -(19:23:36 2015-11-16)[nerv] info: create layer: recurrentL1 -(19:23:36 2015-11-16)[nerv] info: create layer: sigmoidL1 -(19:23:36 2015-11-16)[nerv] info: create layer: combinerL1 -(19:23:36 2015-11-16)[nerv] info: create layer: outputL -(19:23:36 2015-11-16)[nerv] info: create layer: softmaxL -(19:23:36 2015-11-16)[nerv] info: create layer: selectL1 +(00:16:35 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:16:35 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:16:35 2015-11-17)[nerv] info: create layer: combinerL1 +(00:16:35 2015-11-17)[nerv] info: create layer: outputL +(00:16:35 2015-11-17)[nerv] info: create layer: softmaxL +(00:16:35 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER3 LR1]: preparing layers end. [SCHEDULER ITER3 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -86,164 +578,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 3 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER3 LR1]: 40099 words processed Mon Nov 16 19:23:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.401654. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25687 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04683 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43521 clock time -[SCHEDULER ITER3 LR1]: 80063 words processed Mon Nov 16 19:24:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.374433. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19857 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02717 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71469 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42511 clock time -[SCHEDULER ITER3 LR1]: 120068 words processed Mon Nov 16 19:24:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.378173. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31757 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05634 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76081 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43819 clock time -[SCHEDULER ITER3 LR1]: 160017 words processed Mon Nov 16 19:24:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.366419. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21593 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02930 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72744 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42766 clock time -[SCHEDULER ITER3 LR1]: 200138 words processed Mon Nov 16 19:24:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.375391. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06194 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44354 clock time -[SCHEDULER ITER3 LR1]: 240007 words processed Mon Nov 16 19:25:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.374863. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15444 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01010 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71058 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41652 clock time -[SCHEDULER ITER3 LR1]: 280135 words processed Mon Nov 16 19:25:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.371663. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32493 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06209 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77125 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44028 clock time -[SCHEDULER ITER3 LR1]: 320080 words processed Mon Nov 16 19:25:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.372572. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22077 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73036 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42811 clock time -[SCHEDULER ITER3 LR1]: 360059 words processed Mon Nov 16 19:25:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.367750. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30888 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05146 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77331 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43544 clock time -[SCHEDULER ITER3 LR1]: 400021 words processed Mon Nov 16 19:26:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.370284. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19708 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02923 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72020 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42434 clock time -[SCHEDULER ITER3 LR1]: 440102 words processed Mon Nov 16 19:26:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.366085. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22346 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04211 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72356 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43221 clock time -[SCHEDULER ITER3 LR1]: 480051 words processed Mon Nov 16 19:26:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.362942. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21148 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02936 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73411 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42472 clock time -[SCHEDULER ITER3 LR1]: 520093 words processed Mon Nov 16 19:26:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.359600. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26407 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05013 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43525 clock time -[SCHEDULER ITER3 LR1]: 560039 words processed Mon Nov 16 19:27:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.352747. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28053 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43241 clock time -[SCHEDULER ITER3 LR1]: 600112 words processed Mon Nov 16 19:27:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.354185. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28527 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05332 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75468 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43716 clock time -[SCHEDULER ITER3 LR1]: 640076 words processed Mon Nov 16 19:27:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.351938. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17509 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02320 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71112 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42335 clock time -[SCHEDULER ITER3 LR1]: 680026 words processed Mon Nov 16 19:27:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.352402. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19184 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02506 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72222 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42376 clock time -[SCHEDULER ITER3 LR1]: 720133 words processed Mon Nov 16 19:28:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.351590. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22742 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04343 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72700 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43243 clock time -[SCHEDULER ITER3 LR1]: 760048 words processed Mon Nov 16 19:28:20 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.347382. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19520 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02816 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72164 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42459 clock time -[SCHEDULER ITER3 LR1]: 800117 words processed Mon Nov 16 19:28:35 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.346849. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33851 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06692 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77792 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43937 clock time -[SCHEDULER ITER3 LR1]: 840116 words processed Mon Nov 16 19:28:50 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.342472. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32912 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76887 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44276 clock time -[SCHEDULER ITER3 LR1]: 880037 words processed Mon Nov 16 19:29:05 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.341243. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22162 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03318 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72779 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43004 clock time +[SCHEDULER ITER3 LR1]: 40092 words processed Tue Nov 17 00:16:49 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.451944. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24393 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97312 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75603 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43213 clock time +[SCHEDULER ITER3 LR1]: 80099 words processed Tue Nov 17 00:17:03 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.440249. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16343 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94441 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73597 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41776 clock time +[SCHEDULER ITER3 LR1]: 120004 words processed Tue Nov 17 00:17:17 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.429951. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15249 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94180 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72770 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41684 clock time +[SCHEDULER ITER3 LR1]: 160114 words processed Tue Nov 17 00:17:32 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.428219. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19216 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95969 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73701 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42746 clock time +[SCHEDULER ITER3 LR1]: 200066 words processed Tue Nov 17 00:17:47 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.427816. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19681 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96944 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74314 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41611 clock time +[SCHEDULER ITER3 LR1]: 240045 words processed Tue Nov 17 00:18:02 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.421879. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20117 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95415 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75101 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42315 clock time +[SCHEDULER ITER3 LR1]: 280057 words processed Tue Nov 17 00:18:17 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.418622. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26805 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97981 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76757 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43297 clock time +[SCHEDULER ITER3 LR1]: 320106 words processed Tue Nov 17 00:18:32 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.413292. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22311 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96642 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75295 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42966 clock time +[SCHEDULER ITER3 LR1]: 360024 words processed Tue Nov 17 00:18:47 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.410893. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21722 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95975 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75310 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42653 clock time +[SCHEDULER ITER3 LR1]: 400089 words processed Tue Nov 17 00:19:02 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.409763. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33080 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98658 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79905 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44089 clock time +[SCHEDULER ITER3 LR1]: 440067 words processed Tue Nov 17 00:19:17 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.408198. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19522 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95467 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75045 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42259 clock time +[SCHEDULER ITER3 LR1]: 480051 words processed Tue Nov 17 00:19:31 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.406487. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12384 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94097 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71649 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41720 clock time +[SCHEDULER ITER3 LR1]: 520140 words processed Tue Nov 17 00:19:46 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.404808. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19942 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95834 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75314 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42578 clock time +[SCHEDULER ITER3 LR1]: 560132 words processed Tue Nov 17 00:20:01 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.403215. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20221 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95699 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74750 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42386 clock time +[SCHEDULER ITER3 LR1]: 600118 words processed Tue Nov 17 00:20:16 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.404832. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29962 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97894 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78289 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43809 clock time +[SCHEDULER ITER3 LR1]: 640090 words processed Tue Nov 17 00:20:31 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.402722. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21695 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95831 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75755 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42378 clock time +[SCHEDULER ITER3 LR1]: 680075 words processed Tue Nov 17 00:20:45 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.401527. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15485 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94411 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73283 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41815 clock time +[SCHEDULER ITER3 LR1]: 720043 words processed Tue Nov 17 00:20:59 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.400318. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94281 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71709 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41870 clock time +[SCHEDULER ITER3 LR1]: 760012 words processed Tue Nov 17 00:21:14 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.398883. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19500 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95331 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75083 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42308 clock time +[SCHEDULER ITER3 LR1]: 800113 words processed Tue Nov 17 00:21:29 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.397416. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96361 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74747 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42731 clock time +[SCHEDULER ITER3 LR1]: 840089 words processed Tue Nov 17 00:21:43 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.395775. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12880 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94176 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.71968 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41782 clock time +[SCHEDULER ITER3 LR1]: 880052 words processed Tue Nov 17 00:21:58 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.394509. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21553 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95974 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42560 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER3 LR1]: Displaying result: -[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 248.43219795907> <PPL_OOV 219.10997078782> <LOGP -2175853.7789479> +[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 283.19807665492> <PPL_OOV 248.01686534119> <LOGP -2225883.3974227> [SCHEDULER ITER3 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER3 LR1]: shuffling training file ===PEEK ON TEST 3=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER3 LR1]: 40087 words processed Mon Nov 16 19:29:18 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.354254. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78797 clock time +[SCHEDULER ITER3 LR1]: 40087 words processed Tue Nov 17 00:22:09 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.340680. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68311 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75306 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER3 LR1]: Displaying result: -[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 258.77770964542> <PPL_OOV 229.7206912305> <LOGP -194633.7254304> +[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 254.08467494085> <PPL_OOV 220.42020727768> <LOGP -193154.21343403> [SCHEDULER ITER3 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 3=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER3 LR1]: 40095 words processed Mon Nov 16 19:29:31 2015. - [SCHEDULER ITER3 LR1]: log prob per sample :-2.410570. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.71311 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77428 clock time +[SCHEDULER ITER3 LR1]: 40095 words processed Tue Nov 17 00:22:22 2015. + [SCHEDULER ITER3 LR1]: log prob per sample :-2.390081. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67167 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74571 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER3 LR1]: Displaying result: -[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 261.52515353253> <PPL_OOV 237.80638217976> <LOGP -175270.24579595> +[SCHEDULER ITER3 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 259.50985041817> <PPL_OOV 231.23808804042> <LOGP -174373.01911756> [SCHEDULER ITER3 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER3 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.3... @@ -252,27 +744,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1908286 +reading chunk 1 from 1909384 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1913140 +reading chunk 2 from 1914241 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 47991009 +reading chunk 3 from 47979913 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48107236 +reading chunk 4 from 48096187 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94141877 +reading chunk 5 from 94128148 [SCHEDULER ITER4 LR1]: preparing parameters end. [SCHEDULER ITER4 LR1]: preparing layers... -(19:29:42 2015-11-16)[nerv] info: create layer: recurrentL1 -(19:29:42 2015-11-16)[nerv] info: create layer: sigmoidL1 -(19:29:42 2015-11-16)[nerv] info: create layer: combinerL1 -(19:29:42 2015-11-16)[nerv] info: create layer: outputL -(19:29:42 2015-11-16)[nerv] info: create layer: softmaxL -(19:29:42 2015-11-16)[nerv] info: create layer: selectL1 +(00:22:33 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:22:33 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:22:33 2015-11-17)[nerv] info: create layer: combinerL1 +(00:22:33 2015-11-17)[nerv] info: create layer: outputL +(00:22:33 2015-11-17)[nerv] info: create layer: softmaxL +(00:22:33 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER4 LR1]: preparing layers end. [SCHEDULER ITER4 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -300,164 +792,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 4 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER4 LR1]: 40092 words processed Mon Nov 16 19:29:56 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.468880. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13283 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94766 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.68819 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43918 clock time -[SCHEDULER ITER4 LR1]: 80099 words processed Mon Nov 16 19:30:10 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.409142. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.02044 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91168 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64792 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42609 clock time -[SCHEDULER ITER4 LR1]: 120004 words processed Mon Nov 16 19:30:24 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.384936. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.00305 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91040 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63526 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42683 clock time -[SCHEDULER ITER4 LR1]: 160114 words processed Mon Nov 16 19:30:38 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.375498. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.05765 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93086 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.65223 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43543 clock time -[SCHEDULER ITER4 LR1]: 200066 words processed Mon Nov 16 19:30:52 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.369759. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.02876 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91642 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42892 clock time -[SCHEDULER ITER4 LR1]: 240045 words processed Mon Nov 16 19:31:07 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.363138. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.04041 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66035 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42694 clock time -[SCHEDULER ITER4 LR1]: 280057 words processed Mon Nov 16 19:31:21 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.357591. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.06577 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93130 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.65938 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43454 clock time -[SCHEDULER ITER4 LR1]: 320106 words processed Mon Nov 16 19:31:35 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.351439. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.03822 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92729 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64614 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43377 clock time -[SCHEDULER ITER4 LR1]: 360024 words processed Mon Nov 16 19:31:50 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.348008. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.04395 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92032 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66189 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42712 clock time -[SCHEDULER ITER4 LR1]: 400089 words processed Mon Nov 16 19:32:05 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.345590. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16780 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95229 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.70772 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44402 clock time -[SCHEDULER ITER4 LR1]: 440067 words processed Mon Nov 16 19:32:19 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.343815. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.05752 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92189 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66830 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42858 clock time -[SCHEDULER ITER4 LR1]: 480051 words processed Mon Nov 16 19:32:33 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.342301. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.00684 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91463 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64211 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42672 clock time -[SCHEDULER ITER4 LR1]: 520140 words processed Mon Nov 16 19:32:47 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.340726. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.06603 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93166 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66309 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43539 clock time -[SCHEDULER ITER4 LR1]: 560132 words processed Mon Nov 16 19:33:01 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.339017. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.02621 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91591 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.65417 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42811 clock time -[SCHEDULER ITER4 LR1]: 600118 words processed Mon Nov 16 19:33:15 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.337135. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.04052 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91867 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66001 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42870 clock time -[SCHEDULER ITER4 LR1]: 640090 words processed Mon Nov 16 19:33:29 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.335000. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.99942 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.90996 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64063 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42654 clock time -[SCHEDULER ITER4 LR1]: 680075 words processed Mon Nov 16 19:33:43 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.334276. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.97986 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.90818 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63057 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42476 clock time -[SCHEDULER ITER4 LR1]: 720043 words processed Mon Nov 16 19:33:58 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.333732. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.02579 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91550 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.65667 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42625 clock time -[SCHEDULER ITER4 LR1]: 760012 words processed Mon Nov 16 19:34:12 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.332585. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.06318 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92825 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.66598 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43217 clock time -[SCHEDULER ITER4 LR1]: 800113 words processed Mon Nov 16 19:34:27 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.331627. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19833 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95841 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44986 clock time -[SCHEDULER ITER4 LR1]: 840089 words processed Mon Nov 16 19:34:41 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.330192. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:12.99251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91047 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.63615 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42593 clock time -[SCHEDULER ITER4 LR1]: 880052 words processed Mon Nov 16 19:34:55 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.329379. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.01230 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91390 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64582 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42720 clock time +[SCHEDULER ITER4 LR1]: 40092 words processed Tue Nov 17 00:22:47 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.411635. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30488 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98605 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78840 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43755 clock time +[SCHEDULER ITER4 LR1]: 80099 words processed Tue Nov 17 00:23:01 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.396264. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18491 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95328 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74726 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41937 clock time +[SCHEDULER ITER4 LR1]: 120004 words processed Tue Nov 17 00:23:16 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.384882. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22731 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96271 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75796 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42428 clock time +[SCHEDULER ITER4 LR1]: 160114 words processed Tue Nov 17 00:23:31 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.381740. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33553 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99614 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79490 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44209 clock time +[SCHEDULER ITER4 LR1]: 200066 words processed Tue Nov 17 00:23:46 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.379722. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24940 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97050 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77031 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42731 clock time +[SCHEDULER ITER4 LR1]: 240045 words processed Tue Nov 17 00:24:01 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.373056. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24639 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96698 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77245 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42836 clock time +[SCHEDULER ITER4 LR1]: 280057 words processed Tue Nov 17 00:24:16 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.369263. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98078 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77447 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43321 clock time +[SCHEDULER ITER4 LR1]: 320106 words processed Tue Nov 17 00:24:31 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.363696. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00215 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44387 clock time +[SCHEDULER ITER4 LR1]: 360024 words processed Tue Nov 17 00:24:46 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.361212. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23901 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96844 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76681 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42809 clock time +[SCHEDULER ITER4 LR1]: 400089 words processed Tue Nov 17 00:25:01 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.359222. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23928 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97686 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76359 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42989 clock time +[SCHEDULER ITER4 LR1]: 440067 words processed Tue Nov 17 00:25:16 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.357860. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22706 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96520 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76286 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42673 clock time +[SCHEDULER ITER4 LR1]: 480051 words processed Tue Nov 17 00:25:31 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.356565. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31125 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98647 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79505 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43707 clock time +[SCHEDULER ITER4 LR1]: 520140 words processed Tue Nov 17 00:25:46 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.355206. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34890 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00360 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80215 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44646 clock time +[SCHEDULER ITER4 LR1]: 560132 words processed Tue Nov 17 00:26:01 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.353754. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25548 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97148 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77926 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43004 clock time +[SCHEDULER ITER4 LR1]: 600118 words processed Tue Nov 17 00:26:15 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.354322. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17814 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95572 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74528 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42027 clock time +[SCHEDULER ITER4 LR1]: 640090 words processed Tue Nov 17 00:26:29 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.353269. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16963 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95138 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74776 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41862 clock time +[SCHEDULER ITER4 LR1]: 680075 words processed Tue Nov 17 00:26:43 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.352337. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16741 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95087 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74551 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41998 clock time +[SCHEDULER ITER4 LR1]: 720043 words processed Tue Nov 17 00:26:58 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.351371. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18134 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95704 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74577 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42107 clock time +[SCHEDULER ITER4 LR1]: 760012 words processed Tue Nov 17 00:27:13 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.350162. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27103 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97795 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77703 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43423 clock time +[SCHEDULER ITER4 LR1]: 800113 words processed Tue Nov 17 00:27:28 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.348895. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31127 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00287 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78249 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44500 clock time +[SCHEDULER ITER4 LR1]: 840089 words processed Tue Nov 17 00:27:43 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.347213. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20406 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96165 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75661 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42306 clock time +[SCHEDULER ITER4 LR1]: 880052 words processed Tue Nov 17 00:27:58 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.345981. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20763 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96052 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75442 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42431 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER4 LR1]: Displaying result: -[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 241.85074825678> <PPL_OOV 213.49747663731> <LOGP -2165377.8737255> +[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 251.99566219377> <PPL_OOV 221.74821313961> <LOGP -2180685.7714076> [SCHEDULER ITER4 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER4 LR1]: shuffling training file ===PEEK ON TEST 4=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER4 LR1]: 40087 words processed Mon Nov 16 19:35:06 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.298894. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.65216 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.70372 clock time +[SCHEDULER ITER4 LR1]: 40087 words processed Tue Nov 17 00:28:09 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.306434. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69886 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76499 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER4 LR1]: Displaying result: -[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 230.35932079439> <PPL_OOV 200.65005327988> <LOGP -189790.06980019> +[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 231.93289433612> <PPL_OOV 203.34139340474> <LOGP -190267.05230253> [SCHEDULER ITER4 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 4=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER4 LR1]: 40095 words processed Mon Nov 16 19:35:19 2015. - [SCHEDULER ITER4 LR1]: log prob per sample :-2.353715. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.55962 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.64709 clock time +[SCHEDULER ITER4 LR1]: 40095 words processed Tue Nov 17 00:28:22 2015. + [SCHEDULER ITER4 LR1]: log prob per sample :-2.358860. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69384 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76024 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER4 LR1]: Displaying result: -[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 235.61401547839> <PPL_OOV 210.50727974032> <LOGP -171364.18270901> +[SCHEDULER ITER4 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 237.40789525667> <PPL_OOV 213.35176853035> <LOGP -171794.13919305> [SCHEDULER ITER4 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER4 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.4... @@ -466,27 +958,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46037941 +reading chunk 1 from 46035904 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46042798 +reading chunk 2 from 46040761 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92142353 +reading chunk 3 from 92124584 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92258769 +reading chunk 4 from 92240906 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94165007 +reading chunk 5 from 94149353 [SCHEDULER ITER5 LR1]: preparing parameters end. [SCHEDULER ITER5 LR1]: preparing layers... -(19:35:30 2015-11-16)[nerv] info: create layer: recurrentL1 -(19:35:30 2015-11-16)[nerv] info: create layer: sigmoidL1 -(19:35:30 2015-11-16)[nerv] info: create layer: combinerL1 -(19:35:30 2015-11-16)[nerv] info: create layer: outputL -(19:35:30 2015-11-16)[nerv] info: create layer: softmaxL -(19:35:30 2015-11-16)[nerv] info: create layer: selectL1 +(00:28:33 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:28:33 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:28:33 2015-11-17)[nerv] info: create layer: combinerL1 +(00:28:33 2015-11-17)[nerv] info: create layer: outputL +(00:28:33 2015-11-17)[nerv] info: create layer: softmaxL +(00:28:33 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER5 LR1]: preparing layers end. [SCHEDULER ITER5 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -514,193 +1006,193 @@ TNN initing storage outputL -> softmaxL ===ITERATION 5 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER5 LR1]: 40092 words processed Mon Nov 16 19:35:45 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.339645. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32348 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97352 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84007 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44747 clock time -[SCHEDULER ITER5 LR1]: 80099 words processed Mon Nov 16 19:36:00 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.317425. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25965 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94775 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81422 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43733 clock time -[SCHEDULER ITER5 LR1]: 120004 words processed Mon Nov 16 19:36:15 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.305748. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28908 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96200 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81970 clock time +[SCHEDULER ITER5 LR1]: 40092 words processed Tue Nov 17 00:28:47 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.920922. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22756 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96821 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75057 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42902 clock time +[SCHEDULER ITER5 LR1]: 80099 words processed Tue Nov 17 00:29:02 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.660456. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21214 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95997 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74650 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42350 clock time +[SCHEDULER ITER5 LR1]: 120004 words processed Tue Nov 17 00:29:17 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.560737. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27773 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97125 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77687 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43062 clock time +[SCHEDULER ITER5 LR1]: 160114 words processed Tue Nov 17 00:29:32 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.511955. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35829 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99881 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80314 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44084 clock time +[SCHEDULER ITER5 LR1]: 200066 words processed Tue Nov 17 00:29:47 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.481844. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18376 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95244 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74194 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42094 clock time +[SCHEDULER ITER5 LR1]: 240045 words processed Tue Nov 17 00:30:01 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.456659. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16091 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94812 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73178 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41958 clock time +[SCHEDULER ITER5 LR1]: 280057 words processed Tue Nov 17 00:30:16 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.438562. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22194 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96578 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42829 clock time +[SCHEDULER ITER5 LR1]: 320106 words processed Tue Nov 17 00:30:31 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.423035. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34675 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99935 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79088 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44281 clock time +[SCHEDULER ITER5 LR1]: 360024 words processed Tue Nov 17 00:30:46 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.412333. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24196 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96825 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42916 clock time +[SCHEDULER ITER5 LR1]: 400089 words processed Tue Nov 17 00:31:01 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.403296. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22880 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96992 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75435 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42960 clock time +[SCHEDULER ITER5 LR1]: 440067 words processed Tue Nov 17 00:31:16 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.396811. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26098 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43675 clock time +[SCHEDULER ITER5 LR1]: 480051 words processed Tue Nov 17 00:31:31 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.390872. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27622 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98212 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77225 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43878 clock time +[SCHEDULER ITER5 LR1]: 520140 words processed Tue Nov 17 00:31:46 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.385546. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27610 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98854 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76357 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43739 clock time +[SCHEDULER ITER5 LR1]: 560132 words processed Tue Nov 17 00:32:01 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.380535. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18458 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95422 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74174 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41994 clock time +[SCHEDULER ITER5 LR1]: 600118 words processed Tue Nov 17 00:32:15 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.389353. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15921 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94525 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73828 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41825 clock time +[SCHEDULER ITER5 LR1]: 640090 words processed Tue Nov 17 00:32:30 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.391920. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26483 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98412 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76130 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43629 clock time +[SCHEDULER ITER5 LR1]: 680075 words processed Tue Nov 17 00:32:44 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.393276. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15096 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94617 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73040 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41806 clock time +[SCHEDULER ITER5 LR1]: 720043 words processed Tue Nov 17 00:32:58 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.391083. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16223 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95066 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73279 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41836 clock time +[SCHEDULER ITER5 LR1]: 760012 words processed Tue Nov 17 00:33:13 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.388161. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19228 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95890 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74198 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42295 clock time +[SCHEDULER ITER5 LR1]: 800113 words processed Tue Nov 17 00:33:28 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.385378. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33340 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99614 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79299 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44193 clock time -[SCHEDULER ITER5 LR1]: 160114 words processed Mon Nov 16 19:36:29 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.302881. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18222 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94439 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76730 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43387 clock time -[SCHEDULER ITER5 LR1]: 200066 words processed Mon Nov 16 19:36:44 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.300586. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26352 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95203 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80939 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44018 clock time -[SCHEDULER ITER5 LR1]: 240045 words processed Mon Nov 16 19:36:59 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.294586. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15493 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92808 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77520 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42416 clock time -[SCHEDULER ITER5 LR1]: 280057 words processed Mon Nov 16 19:37:14 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.290372. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21407 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94570 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79555 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43495 clock time -[SCHEDULER ITER5 LR1]: 320106 words processed Mon Nov 16 19:37:28 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.285735. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16169 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94020 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76253 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43204 clock time -[SCHEDULER ITER5 LR1]: 360024 words processed Mon Nov 16 19:37:42 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.283445. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15456 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93256 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76159 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42953 clock time -[SCHEDULER ITER5 LR1]: 400089 words processed Mon Nov 16 19:37:57 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.281396. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18383 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94399 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77670 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43374 clock time -[SCHEDULER ITER5 LR1]: 440067 words processed Mon Nov 16 19:38:12 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.280582. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14002 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92845 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76492 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42464 clock time -[SCHEDULER ITER5 LR1]: 480051 words processed Mon Nov 16 19:38:27 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.280046. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18139 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79444 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42539 clock time -[SCHEDULER ITER5 LR1]: 520140 words processed Mon Nov 16 19:38:42 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.279298. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17428 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94212 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77082 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43341 clock time -[SCHEDULER ITER5 LR1]: 560132 words processed Mon Nov 16 19:38:57 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.278341. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17924 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93452 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78455 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43026 clock time -[SCHEDULER ITER5 LR1]: 600118 words processed Mon Nov 16 19:39:12 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.277074. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27450 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95277 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82452 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43800 clock time -[SCHEDULER ITER5 LR1]: 640090 words processed Mon Nov 16 19:39:27 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.275369. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16229 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93076 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78169 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42522 clock time -[SCHEDULER ITER5 LR1]: 680075 words processed Mon Nov 16 19:39:42 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.275114. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15877 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92924 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78210 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42575 clock time -[SCHEDULER ITER5 LR1]: 720043 words processed Mon Nov 16 19:39:56 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.274860. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12358 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92467 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75768 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42370 clock time -[SCHEDULER ITER5 LR1]: 760012 words processed Mon Nov 16 19:40:11 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.274031. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92838 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77272 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42630 clock time -[SCHEDULER ITER5 LR1]: 800113 words processed Mon Nov 16 19:40:26 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.273616. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18233 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94496 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77598 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43450 clock time -[SCHEDULER ITER5 LR1]: 840089 words processed Mon Nov 16 19:40:41 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.272619. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21063 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94888 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78385 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43517 clock time -[SCHEDULER ITER5 LR1]: 880052 words processed Mon Nov 16 19:40:56 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.272217. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16136 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92998 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78300 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42459 clock time +[SCHEDULER ITER5 LR1]: 840089 words processed Tue Nov 17 00:33:43 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.382788. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26624 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97510 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77150 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43145 clock time +[SCHEDULER ITER5 LR1]: 880052 words processed Tue Nov 17 00:33:57 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.380219. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15630 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94733 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73162 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41864 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER5 LR1]: Displaying result: -[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 211.34935677039> <PPL_OOV 187.19131095214> <LOGP -2112291.7783632> +[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 269.55389105673> <PPL_OOV 239.78923630153> <LOGP -2212263.496602> [SCHEDULER ITER5 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER5 LR1]: shuffling training file ===PEEK ON TEST 5=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER5 LR1]: 40087 words processed Mon Nov 16 19:41:07 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.266882. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70246 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78859 clock time +[SCHEDULER ITER5 LR1]: 40087 words processed Tue Nov 17 00:34:08 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.312602. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74532 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER5 LR1]: Displaying result: -[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 211.88666827405> <PPL_OOV 185.90353406678> <LOGP -187057.38065016> +[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 237.31440353483> <PPL_OOV 206.64068876902> <LOGP -190843.24175269> [SCHEDULER ITER5 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 5=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER5 LR1]: 40095 words processed Mon Nov 16 19:41:20 2015. - [SCHEDULER ITER5 LR1]: log prob per sample :-2.323979. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72302 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79517 clock time +[SCHEDULER ITER5 LR1]: 40095 words processed Tue Nov 17 00:34:21 2015. + [SCHEDULER ITER5 LR1]: log prob per sample :-2.364799. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67575 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74965 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER5 LR1]: Displaying result: -[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 216.72667354494> <PPL_OOV 194.77072809761> <LOGP -168875.2672087> +[SCHEDULER ITER5 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 243.08905084179> <PPL_OOV 217.3272089236> <LOGP -172385.53620102> [SCHEDULER ITER5 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER5 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.5... +[SCHEDULER ITER5 LR1]: PPL did not improve, rejected, copying param file of last iter... [SCHEDULER ITER6 LR1]: preparing parameters... [SCHEDULER ITER6 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.5... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 1904838 +reading chunk 1 from 46035904 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1909695 +reading chunk 2 from 46040761 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48027406 +reading chunk 3 from 92124584 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48143739 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 92240906 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94185248 +reading chunk 5 from 94149353 [SCHEDULER ITER6 LR1]: preparing parameters end. [SCHEDULER ITER6 LR1]: preparing layers... -(19:41:31 2015-11-16)[nerv] info: create layer: recurrentL1 -(19:41:31 2015-11-16)[nerv] info: create layer: sigmoidL1 -(19:41:31 2015-11-16)[nerv] info: create layer: combinerL1 -(19:41:31 2015-11-16)[nerv] info: create layer: outputL -(19:41:31 2015-11-16)[nerv] info: create layer: softmaxL -(19:41:31 2015-11-16)[nerv] info: create layer: selectL1 +(00:34:31 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:34:31 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:34:31 2015-11-17)[nerv] info: create layer: combinerL1 +(00:34:31 2015-11-17)[nerv] info: create layer: outputL +(00:34:31 2015-11-17)[nerv] info: create layer: softmaxL +(00:34:31 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER6 LR1]: preparing layers end. [SCHEDULER ITER6 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -728,193 +1220,193 @@ TNN initing storage outputL -> softmaxL ===ITERATION 6 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER6 LR1]: 40092 words processed Mon Nov 16 19:41:46 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.262859. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30509 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95829 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82954 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44497 clock time -[SCHEDULER ITER6 LR1]: 80099 words processed Mon Nov 16 19:42:01 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.256861. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94976 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81911 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44514 clock time -[SCHEDULER ITER6 LR1]: 120004 words processed Mon Nov 16 19:42:16 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.251430. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14338 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91591 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76243 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42779 clock time -[SCHEDULER ITER6 LR1]: 160114 words processed Mon Nov 16 19:42:31 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.252258. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25146 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94930 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79763 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44322 clock time -[SCHEDULER ITER6 LR1]: 200066 words processed Mon Nov 16 19:42:46 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.252740. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24557 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94297 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79544 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44001 clock time -[SCHEDULER ITER6 LR1]: 240045 words processed Mon Nov 16 19:43:00 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.248501. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13878 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92118 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75799 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42859 clock time -[SCHEDULER ITER6 LR1]: 280057 words processed Mon Nov 16 19:43:15 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.245658. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18186 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93093 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77498 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43733 clock time -[SCHEDULER ITER6 LR1]: 320106 words processed Mon Nov 16 19:43:30 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.242452. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23852 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94407 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80638 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43804 clock time -[SCHEDULER ITER6 LR1]: 360024 words processed Mon Nov 16 19:43:45 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.241240. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21118 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93112 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80305 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43131 clock time -[SCHEDULER ITER6 LR1]: 400089 words processed Mon Nov 16 19:44:00 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.239991. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26483 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94944 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81150 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44485 clock time -[SCHEDULER ITER6 LR1]: 440067 words processed Mon Nov 16 19:44:15 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.240005. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25938 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95053 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80400 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44342 clock time -[SCHEDULER ITER6 LR1]: 480051 words processed Mon Nov 16 19:44:30 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.240189. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22771 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94556 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78703 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43955 clock time -[SCHEDULER ITER6 LR1]: 520140 words processed Mon Nov 16 19:44:45 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.240002. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29153 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81080 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45685 clock time -[SCHEDULER ITER6 LR1]: 560132 words processed Mon Nov 16 19:45:00 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.239520. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93829 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79051 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43733 clock time -[SCHEDULER ITER6 LR1]: 600118 words processed Mon Nov 16 19:45:15 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.238317. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31952 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95527 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84495 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44292 clock time -[SCHEDULER ITER6 LR1]: 640090 words processed Mon Nov 16 19:45:30 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.236891. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15653 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92046 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77497 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42729 clock time -[SCHEDULER ITER6 LR1]: 680075 words processed Mon Nov 16 19:45:45 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.237217. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13630 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91738 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76375 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42763 clock time -[SCHEDULER ITER6 LR1]: 720043 words processed Mon Nov 16 19:46:00 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.237386. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25722 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94448 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80934 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43601 clock time -[SCHEDULER ITER6 LR1]: 760012 words processed Mon Nov 16 19:46:15 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.237187. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21321 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95477 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77299 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44531 clock time -[SCHEDULER ITER6 LR1]: 800113 words processed Mon Nov 16 19:46:30 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.237168. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25265 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97044 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79578 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43598 clock time -[SCHEDULER ITER6 LR1]: 840089 words processed Mon Nov 16 19:46:45 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.236424. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23587 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80866 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43496 clock time -[SCHEDULER ITER6 LR1]: 880052 words processed Mon Nov 16 19:47:00 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.236239. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20416 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92983 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79912 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43148 clock time +[SCHEDULER ITER6 LR1]: 40092 words processed Tue Nov 17 00:34:45 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.996969. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32205 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98973 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78732 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44429 clock time +[SCHEDULER ITER6 LR1]: 80099 words processed Tue Nov 17 00:35:00 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.704185. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28977 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97386 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78145 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43591 clock time +[SCHEDULER ITER6 LR1]: 120004 words processed Tue Nov 17 00:35:15 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.591836. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18920 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95332 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74157 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42466 clock time +[SCHEDULER ITER6 LR1]: 160114 words processed Tue Nov 17 00:35:30 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.536426. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20163 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96414 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74297 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42791 clock time +[SCHEDULER ITER6 LR1]: 200066 words processed Tue Nov 17 00:35:45 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.502609. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22194 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97505 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74875 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42507 clock time +[SCHEDULER ITER6 LR1]: 240045 words processed Tue Nov 17 00:36:00 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.473995. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22544 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95899 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76286 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42695 clock time +[SCHEDULER ITER6 LR1]: 280057 words processed Tue Nov 17 00:36:15 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.453613. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23177 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97226 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75371 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43188 clock time +[SCHEDULER ITER6 LR1]: 320106 words processed Tue Nov 17 00:36:30 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.435705. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27344 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98545 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76872 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43841 clock time +[SCHEDULER ITER6 LR1]: 360024 words processed Tue Nov 17 00:36:45 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.423627. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29991 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97881 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78618 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43762 clock time +[SCHEDULER ITER6 LR1]: 400089 words processed Tue Nov 17 00:37:00 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.413293. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98822 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78680 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44183 clock time +[SCHEDULER ITER6 LR1]: 440067 words processed Tue Nov 17 00:37:14 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.405194. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17583 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95265 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74339 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42163 clock time +[SCHEDULER ITER6 LR1]: 480051 words processed Tue Nov 17 00:37:29 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.398437. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22457 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96424 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76022 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42621 clock time +[SCHEDULER ITER6 LR1]: 520140 words processed Tue Nov 17 00:37:44 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.392212. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22752 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97106 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75333 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43438 clock time +[SCHEDULER ITER6 LR1]: 560132 words processed Tue Nov 17 00:37:58 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.386436. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14672 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94424 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73399 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41900 clock time +[SCHEDULER ITER6 LR1]: 600118 words processed Tue Nov 17 00:38:12 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.384202. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16042 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73843 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42047 clock time +[SCHEDULER ITER6 LR1]: 640090 words processed Tue Nov 17 00:38:26 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.380705. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15965 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94523 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73916 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42132 clock time +[SCHEDULER ITER6 LR1]: 680075 words processed Tue Nov 17 00:38:41 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.377167. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29248 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97952 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78776 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43438 clock time +[SCHEDULER ITER6 LR1]: 720043 words processed Tue Nov 17 00:38:56 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.373437. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21865 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96220 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75751 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42865 clock time +[SCHEDULER ITER6 LR1]: 760012 words processed Tue Nov 17 00:39:11 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.369611. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17506 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95270 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74192 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42277 clock time +[SCHEDULER ITER6 LR1]: 800113 words processed Tue Nov 17 00:39:26 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.366322. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28833 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98265 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78402 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43971 clock time +[SCHEDULER ITER6 LR1]: 840089 words processed Tue Nov 17 00:39:40 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.363158. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14517 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94559 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72922 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41989 clock time +[SCHEDULER ITER6 LR1]: 880052 words processed Tue Nov 17 00:39:55 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.360303. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22453 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96385 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75714 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43101 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER6 LR1]: Displaying result: -[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 193.89383590358> <PPL_OOV 172.33079535956> <LOGP -2078898.3448018> +[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 257.91914058884> <PPL_OOV 229.016965913> <LOGP -2193707.0151282> [SCHEDULER ITER6 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER6 LR1]: shuffling training file ===PEEK ON TEST 6=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER6 LR1]: 40087 words processed Mon Nov 16 19:47:11 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.245922. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70007 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77955 clock time +[SCHEDULER ITER6 LR1]: 40087 words processed Tue Nov 17 00:40:06 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.293396. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69484 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76237 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER6 LR1]: Displaying result: -[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 202.3013471096> <PPL_OOV 177.03896783534> <LOGP -185308.31686527> +[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 226.73661220959> <PPL_OOV 197.45240087571> <LOGP -189214.96718214> [SCHEDULER ITER6 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 6=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER6 LR1]: 40095 words processed Mon Nov 16 19:47:25 2015. - [SCHEDULER ITER6 LR1]: log prob per sample :-2.304926. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70143 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78097 clock time +[SCHEDULER ITER6 LR1]: 40095 words processed Tue Nov 17 00:40:19 2015. + [SCHEDULER ITER6 LR1]: log prob per sample :-2.348844. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69175 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75979 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER6 LR1]: Displaying result: -[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 207.07732688932> <PPL_OOV 185.75783021617> <LOGP -167357.54027598> +[SCHEDULER ITER6 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 232.93335170611> <PPL_OOV 208.23433692985> <LOGP -171016.42091829> [SCHEDULER ITER6 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER6 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.6... [SCHEDULER ITER7 LR1]: preparing parameters... [SCHEDULER ITER7 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.6... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 46043903 +reading chunk 1 from 1907214 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46048759 +reading chunk 2 from 1912074 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92184772 +reading chunk 3 from 48011988 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92301105 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 48128347 +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94204599 +reading chunk 5 from 94168656 [SCHEDULER ITER7 LR1]: preparing parameters end. [SCHEDULER ITER7 LR1]: preparing layers... -(19:47:36 2015-11-16)[nerv] info: create layer: recurrentL1 -(19:47:36 2015-11-16)[nerv] info: create layer: sigmoidL1 -(19:47:36 2015-11-16)[nerv] info: create layer: combinerL1 -(19:47:36 2015-11-16)[nerv] info: create layer: outputL -(19:47:36 2015-11-16)[nerv] info: create layer: softmaxL -(19:47:36 2015-11-16)[nerv] info: create layer: selectL1 +(00:40:30 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:40:30 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:40:30 2015-11-17)[nerv] info: create layer: combinerL1 +(00:40:30 2015-11-17)[nerv] info: create layer: outputL +(00:40:30 2015-11-17)[nerv] info: create layer: softmaxL +(00:40:30 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER7 LR1]: preparing layers end. [SCHEDULER ITER7 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -942,193 +1434,193 @@ TNN initing storage outputL -> softmaxL ===ITERATION 7 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER7 LR1]: 40092 words processed Mon Nov 16 19:47:50 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.225906. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21939 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95368 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78896 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43900 clock time -[SCHEDULER ITER7 LR1]: 80099 words processed Mon Nov 16 19:48:05 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.224109. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16155 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93137 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77312 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42438 clock time -[SCHEDULER ITER7 LR1]: 120004 words processed Mon Nov 16 19:48:20 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.219821. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15757 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93223 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42718 clock time -[SCHEDULER ITER7 LR1]: 160114 words processed Mon Nov 16 19:48:35 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.222543. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26964 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96259 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81857 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43736 clock time -[SCHEDULER ITER7 LR1]: 200066 words processed Mon Nov 16 19:48:50 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.223103. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27159 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95979 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81056 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43991 clock time -[SCHEDULER ITER7 LR1]: 240045 words processed Mon Nov 16 19:49:05 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.220590. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23336 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95316 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78952 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43981 clock time -[SCHEDULER ITER7 LR1]: 280057 words processed Mon Nov 16 19:49:20 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.218902. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25931 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95635 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81907 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43567 clock time -[SCHEDULER ITER7 LR1]: 320106 words processed Mon Nov 16 19:49:35 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.215759. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35599 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98062 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85913 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44617 clock time -[SCHEDULER ITER7 LR1]: 360024 words processed Mon Nov 16 19:49:50 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.214424. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17280 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93334 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78062 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42810 clock time -[SCHEDULER ITER7 LR1]: 400089 words processed Mon Nov 16 19:50:05 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.212825. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21202 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95364 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79232 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43465 clock time -[SCHEDULER ITER7 LR1]: 440067 words processed Mon Nov 16 19:50:19 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.212838. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13401 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93181 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75747 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42555 clock time -[SCHEDULER ITER7 LR1]: 480051 words processed Mon Nov 16 19:50:34 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.213009. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22848 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95167 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79588 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43585 clock time -[SCHEDULER ITER7 LR1]: 520140 words processed Mon Nov 16 19:50:49 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.212825. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32416 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98163 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82410 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44985 clock time -[SCHEDULER ITER7 LR1]: 560132 words processed Mon Nov 16 19:51:04 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.212440. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28044 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81380 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44426 clock time -[SCHEDULER ITER7 LR1]: 600118 words processed Mon Nov 16 19:51:19 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.211160. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22035 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95101 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78978 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43636 clock time -[SCHEDULER ITER7 LR1]: 640090 words processed Mon Nov 16 19:51:33 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.209800. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13488 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92897 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76205 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42564 clock time -[SCHEDULER ITER7 LR1]: 680075 words processed Mon Nov 16 19:51:48 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.210009. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28582 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95791 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83462 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43566 clock time -[SCHEDULER ITER7 LR1]: 720043 words processed Mon Nov 16 19:52:02 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.210212. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13837 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93115 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42450 clock time -[SCHEDULER ITER7 LR1]: 760012 words processed Mon Nov 16 19:52:16 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.209773. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12402 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75374 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42400 clock time -[SCHEDULER ITER7 LR1]: 800113 words processed Mon Nov 16 19:52:31 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.209791. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23713 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95444 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81132 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43479 clock time -[SCHEDULER ITER7 LR1]: 840089 words processed Mon Nov 16 19:52:46 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.208838. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24516 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95193 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81845 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43236 clock time -[SCHEDULER ITER7 LR1]: 880052 words processed Mon Nov 16 19:53:00 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.208673. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12365 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92807 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75664 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42380 clock time +[SCHEDULER ITER7 LR1]: 40092 words processed Tue Nov 17 00:40:44 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.341850. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26078 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98260 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76444 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43311 clock time +[SCHEDULER ITER7 LR1]: 80099 words processed Tue Nov 17 00:40:59 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.329983. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28665 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97957 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77753 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43068 clock time +[SCHEDULER ITER7 LR1]: 120004 words processed Tue Nov 17 00:41:14 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.314827. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28194 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98235 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77325 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42988 clock time +[SCHEDULER ITER7 LR1]: 160114 words processed Tue Nov 17 00:41:29 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.310491. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98354 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76448 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43414 clock time +[SCHEDULER ITER7 LR1]: 200066 words processed Tue Nov 17 00:41:44 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.307637. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29216 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98910 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77616 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43081 clock time +[SCHEDULER ITER7 LR1]: 240045 words processed Tue Nov 17 00:41:59 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.300474. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29781 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98342 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78555 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43250 clock time +[SCHEDULER ITER7 LR1]: 280057 words processed Tue Nov 17 00:42:14 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.296762. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38356 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00605 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80943 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44908 clock time +[SCHEDULER ITER7 LR1]: 320106 words processed Tue Nov 17 00:42:29 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.291465. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35504 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00324 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79510 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44744 clock time +[SCHEDULER ITER7 LR1]: 360024 words processed Tue Nov 17 00:42:44 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.289110. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30603 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98514 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78794 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43428 clock time +[SCHEDULER ITER7 LR1]: 400089 words processed Tue Nov 17 00:42:59 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.287066. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76608 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43374 clock time +[SCHEDULER ITER7 LR1]: 440067 words processed Tue Nov 17 00:43:14 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.285858. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16835 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95557 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73435 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42046 clock time +[SCHEDULER ITER7 LR1]: 480051 words processed Tue Nov 17 00:43:29 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.284994. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25252 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97519 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77053 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42934 clock time +[SCHEDULER ITER7 LR1]: 520140 words processed Tue Nov 17 00:43:44 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.284015. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30542 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99397 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43868 clock time +[SCHEDULER ITER7 LR1]: 560132 words processed Tue Nov 17 00:43:59 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.282563. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20366 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96480 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74936 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42468 clock time +[SCHEDULER ITER7 LR1]: 600118 words processed Tue Nov 17 00:44:14 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.282110. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21616 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96725 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75407 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42538 clock time +[SCHEDULER ITER7 LR1]: 640090 words processed Tue Nov 17 00:44:29 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.280363. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17577 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95600 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73931 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42084 clock time +[SCHEDULER ITER7 LR1]: 680075 words processed Tue Nov 17 00:44:44 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.279790. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24122 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76221 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42865 clock time +[SCHEDULER ITER7 LR1]: 720043 words processed Tue Nov 17 00:44:59 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.278987. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27186 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98211 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77115 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43259 clock time +[SCHEDULER ITER7 LR1]: 760012 words processed Tue Nov 17 00:45:14 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.277796. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21569 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74938 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42464 clock time +[SCHEDULER ITER7 LR1]: 800113 words processed Tue Nov 17 00:45:29 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.276979. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31001 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99838 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78180 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43909 clock time +[SCHEDULER ITER7 LR1]: 840089 words processed Tue Nov 17 00:45:44 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.275680. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29525 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98651 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78137 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43498 clock time +[SCHEDULER ITER7 LR1]: 880052 words processed Tue Nov 17 00:45:59 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.274882. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21515 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96619 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75276 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42342 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER7 LR1]: Displaying result: -[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 181.46365030299> <PPL_OOV 161.72625619922> <LOGP -2053258.0870607> +[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 212.72946455377> <PPL_OOV 188.28801239072> <LOGP -2114650.1318705> [SCHEDULER ITER7 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER7 LR1]: shuffling training file ===PEEK ON TEST 7=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER7 LR1]: 40087 words processed Mon Nov 16 19:53:11 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.232519. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73278 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80291 clock time +[SCHEDULER ITER7 LR1]: 40087 words processed Tue Nov 17 00:46:10 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.265946. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69467 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76137 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER7 LR1]: Displaying result: -[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 196.00372584658> <PPL_OOV 171.15770244466> <LOGP -184098.86921264> +[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 212.8779029808> <PPL_OOV 185.08741194959> <LOGP -186899.87644838> [SCHEDULER ITER7 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 7=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER7 LR1]: 40095 words processed Mon Nov 16 19:53:25 2015. - [SCHEDULER ITER7 LR1]: log prob per sample :-2.291901. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81498 clock time +[SCHEDULER ITER7 LR1]: 40095 words processed Tue Nov 17 00:46:23 2015. + [SCHEDULER ITER7 LR1]: log prob per sample :-2.323616. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68426 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75420 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER7 LR1]: Displaying result: -[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 201.17781109485> <PPL_OOV 180.16302699681> <LOGP -166377.89981932> +[SCHEDULER ITER7 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 218.89913931037> <PPL_OOV 195.405644371> <LOGP -168979.52081579> [SCHEDULER ITER7 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER7 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.7... [SCHEDULER ITER8 LR1]: preparing parameters... [SCHEDULER ITER8 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.7... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 1902204 +reading chunk 1 from 46044145 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1907060 +reading chunk 2 from 46049004 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48060548 +reading chunk 3 from 92163803 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48176852 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 92280184 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94221898 +reading chunk 5 from 94186293 [SCHEDULER ITER8 LR1]: preparing parameters end. [SCHEDULER ITER8 LR1]: preparing layers... -(19:53:36 2015-11-16)[nerv] info: create layer: recurrentL1 -(19:53:36 2015-11-16)[nerv] info: create layer: sigmoidL1 -(19:53:36 2015-11-16)[nerv] info: create layer: combinerL1 -(19:53:36 2015-11-16)[nerv] info: create layer: outputL -(19:53:36 2015-11-16)[nerv] info: create layer: softmaxL -(19:53:36 2015-11-16)[nerv] info: create layer: selectL1 +(00:46:34 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:46:34 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:46:34 2015-11-17)[nerv] info: create layer: combinerL1 +(00:46:34 2015-11-17)[nerv] info: create layer: outputL +(00:46:34 2015-11-17)[nerv] info: create layer: softmaxL +(00:46:34 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER8 LR1]: preparing layers end. [SCHEDULER ITER8 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -1156,193 +1648,193 @@ TNN initing storage outputL -> softmaxL ===ITERATION 8 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER8 LR1]: 40092 words processed Mon Nov 16 19:53:50 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.207277. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18732 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94024 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76316 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43873 clock time -[SCHEDULER ITER8 LR1]: 80099 words processed Mon Nov 16 19:54:05 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.205057. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23115 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80346 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43450 clock time -[SCHEDULER ITER8 LR1]: 120004 words processed Mon Nov 16 19:54:20 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.199802. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21246 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43447 clock time -[SCHEDULER ITER8 LR1]: 160114 words processed Mon Nov 16 19:54:35 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.200895. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16965 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93339 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76259 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43486 clock time -[SCHEDULER ITER8 LR1]: 200066 words processed Mon Nov 16 19:54:50 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.200972. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13680 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91749 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75626 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42786 clock time -[SCHEDULER ITER8 LR1]: 240045 words processed Mon Nov 16 19:55:05 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.196588. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15767 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92151 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76829 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42881 clock time -[SCHEDULER ITER8 LR1]: 280057 words processed Mon Nov 16 19:55:20 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.193553. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20250 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93715 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78089 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43662 clock time -[SCHEDULER ITER8 LR1]: 320106 words processed Mon Nov 16 19:55:35 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.190773. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23774 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94573 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79677 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43894 clock time -[SCHEDULER ITER8 LR1]: 360024 words processed Mon Nov 16 19:55:50 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.189543. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95124 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82161 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44773 clock time -[SCHEDULER ITER8 LR1]: 400089 words processed Mon Nov 16 19:56:05 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.188180. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28820 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95822 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82554 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44407 clock time -[SCHEDULER ITER8 LR1]: 440067 words processed Mon Nov 16 19:56:20 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.188437. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20832 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93312 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79124 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43457 clock time -[SCHEDULER ITER8 LR1]: 480051 words processed Mon Nov 16 19:56:35 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.188641. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94878 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75618 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42725 clock time -[SCHEDULER ITER8 LR1]: 520140 words processed Mon Nov 16 19:56:50 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.188369. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23302 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00438 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75628 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43583 clock time -[SCHEDULER ITER8 LR1]: 560132 words processed Mon Nov 16 19:57:04 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.187960. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18137 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98334 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74087 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42653 clock time -[SCHEDULER ITER8 LR1]: 600118 words processed Mon Nov 16 19:57:19 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.187648. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17801 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92123 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76644 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45502 clock time -[SCHEDULER ITER8 LR1]: 640090 words processed Mon Nov 16 19:57:33 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.186406. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.10467 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91617 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73767 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42794 clock time -[SCHEDULER ITER8 LR1]: 680075 words processed Mon Nov 16 19:57:47 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.186378. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.10713 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91476 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74419 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42650 clock time -[SCHEDULER ITER8 LR1]: 720043 words processed Mon Nov 16 19:58:01 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.186701. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.11037 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91643 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74266 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42664 clock time -[SCHEDULER ITER8 LR1]: 760012 words processed Mon Nov 16 19:58:16 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.186947. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12661 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91893 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75104 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42796 clock time -[SCHEDULER ITER8 LR1]: 800113 words processed Mon Nov 16 19:58:31 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.187097. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21660 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94289 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78433 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43984 clock time -[SCHEDULER ITER8 LR1]: 840089 words processed Mon Nov 16 19:58:46 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.186248. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19683 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93566 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78285 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43378 clock time -[SCHEDULER ITER8 LR1]: 880052 words processed Mon Nov 16 19:59:01 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.186156. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94768 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79334 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44486 clock time +[SCHEDULER ITER8 LR1]: 40092 words processed Tue Nov 17 00:46:48 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.316335. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27466 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98044 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77598 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43763 clock time +[SCHEDULER ITER8 LR1]: 80099 words processed Tue Nov 17 00:47:03 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.300821. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18913 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95083 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74990 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42111 clock time +[SCHEDULER ITER8 LR1]: 120004 words processed Tue Nov 17 00:47:18 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.287705. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19620 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95216 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75231 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41989 clock time +[SCHEDULER ITER8 LR1]: 160114 words processed Tue Nov 17 00:47:33 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.283589. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32986 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99408 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79217 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44072 clock time +[SCHEDULER ITER8 LR1]: 200066 words processed Tue Nov 17 00:47:48 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.280939. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29808 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98045 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78980 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43460 clock time +[SCHEDULER ITER8 LR1]: 240045 words processed Tue Nov 17 00:48:03 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.274292. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30978 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98146 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79390 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43793 clock time +[SCHEDULER ITER8 LR1]: 280057 words processed Tue Nov 17 00:48:18 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.269353. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30111 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98645 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78578 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44008 clock time +[SCHEDULER ITER8 LR1]: 320106 words processed Tue Nov 17 00:48:33 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.263944. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33175 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99524 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79879 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44326 clock time +[SCHEDULER ITER8 LR1]: 360024 words processed Tue Nov 17 00:48:48 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.261159. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24423 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96516 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77323 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42981 clock time +[SCHEDULER ITER8 LR1]: 400089 words processed Tue Nov 17 00:49:03 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.258812. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23621 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97140 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76697 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43036 clock time +[SCHEDULER ITER8 LR1]: 440067 words processed Tue Nov 17 00:49:18 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.257714. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22059 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96491 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76223 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42638 clock time +[SCHEDULER ITER8 LR1]: 480051 words processed Tue Nov 17 00:49:33 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.256910. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30096 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97927 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79375 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43478 clock time +[SCHEDULER ITER8 LR1]: 520140 words processed Tue Nov 17 00:49:48 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.255546. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29528 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98474 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78818 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43672 clock time +[SCHEDULER ITER8 LR1]: 560132 words processed Tue Nov 17 00:50:03 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.254077. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96493 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76978 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42774 clock time +[SCHEDULER ITER8 LR1]: 600118 words processed Tue Nov 17 00:50:18 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.253047. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22037 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96402 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76224 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42517 clock time +[SCHEDULER ITER8 LR1]: 640090 words processed Tue Nov 17 00:50:33 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.251071. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20498 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95656 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76094 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42413 clock time +[SCHEDULER ITER8 LR1]: 680075 words processed Tue Nov 17 00:50:47 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.250758. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18378 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95755 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74722 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42123 clock time +[SCHEDULER ITER8 LR1]: 720043 words processed Tue Nov 17 00:51:02 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.250138. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96429 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75949 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42642 clock time +[SCHEDULER ITER8 LR1]: 760012 words processed Tue Nov 17 00:51:17 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.249059. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18493 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95284 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75154 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42271 clock time +[SCHEDULER ITER8 LR1]: 800113 words processed Tue Nov 17 00:51:32 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.248428. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25252 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97577 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77341 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43191 clock time +[SCHEDULER ITER8 LR1]: 840089 words processed Tue Nov 17 00:51:47 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.247396. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26816 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97331 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77880 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43125 clock time +[SCHEDULER ITER8 LR1]: 880052 words processed Tue Nov 17 00:52:02 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.246754. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18564 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95222 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75247 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42000 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER8 LR1]: Displaying result: -[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 171.90531862971> <PPL_OOV 153.56059495327> <LOGP -2032341.6400566> +[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 198.79223504817> <PPL_OOV 176.50384933845> <LOGP -2088557.9726143> [SCHEDULER ITER8 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER8 LR1]: shuffling training file ===PEEK ON TEST 8=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER8 LR1]: 40087 words processed Mon Nov 16 19:59:13 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.223354. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.71911 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79349 clock time +[SCHEDULER ITER8 LR1]: 40087 words processed Tue Nov 17 00:52:13 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.254081. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70240 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77044 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER8 LR1]: Displaying result: -[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 191.60219635223> <PPL_OOV 167.44708240131> <LOGP -183314.22931448> +[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 206.80025315917> <PPL_OOV 180.18006822688> <LOGP -185937.90715788> [SCHEDULER ITER8 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 8=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER8 LR1]: 40095 words processed Mon Nov 16 19:59:26 2015. - [SCHEDULER ITER8 LR1]: log prob per sample :-2.283762. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69871 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77933 clock time +[SCHEDULER ITER8 LR1]: 40095 words processed Tue Nov 17 00:52:26 2015. + [SCHEDULER ITER8 LR1]: log prob per sample :-2.313404. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68474 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75789 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER8 LR1]: Displaying result: -[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 197.01816659556> <PPL_OOV 176.5482345796> <LOGP -165728.64284451> +[SCHEDULER ITER8 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 213.1727932473> <PPL_OOV 190.55635181414> <LOGP -168174.52817362> [SCHEDULER ITER8 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER8 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.8... [SCHEDULER ITER9 LR1]: preparing parameters... [SCHEDULER ITER9 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.8... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 46045738 +reading chunk 1 from 1905077 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46050594 +reading chunk 2 from 1909937 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92220262 +reading chunk 3 from 48040787 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92336581 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 48157200 +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94237739 +reading chunk 5 from 94204508 [SCHEDULER ITER9 LR1]: preparing parameters end. [SCHEDULER ITER9 LR1]: preparing layers... -(19:59:37 2015-11-16)[nerv] info: create layer: recurrentL1 -(19:59:37 2015-11-16)[nerv] info: create layer: sigmoidL1 -(19:59:37 2015-11-16)[nerv] info: create layer: combinerL1 -(19:59:37 2015-11-16)[nerv] info: create layer: outputL -(19:59:37 2015-11-16)[nerv] info: create layer: softmaxL -(19:59:37 2015-11-16)[nerv] info: create layer: selectL1 +(00:52:37 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:52:37 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:52:37 2015-11-17)[nerv] info: create layer: combinerL1 +(00:52:37 2015-11-17)[nerv] info: create layer: outputL +(00:52:37 2015-11-17)[nerv] info: create layer: softmaxL +(00:52:37 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER9 LR1]: preparing layers end. [SCHEDULER ITER9 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -1370,193 +1862,193 @@ TNN initing storage outputL -> softmaxL ===ITERATION 9 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER9 LR1]: 40092 words processed Mon Nov 16 19:59:52 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.180199. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26708 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96292 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81302 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43888 clock time -[SCHEDULER ITER9 LR1]: 80099 words processed Mon Nov 16 20:00:06 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.178767. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13075 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92752 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74942 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42306 clock time -[SCHEDULER ITER9 LR1]: 120004 words processed Mon Nov 16 20:00:21 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.174655. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16473 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93227 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76749 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42656 clock time -[SCHEDULER ITER9 LR1]: 160114 words processed Mon Nov 16 20:00:36 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.176386. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25677 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96181 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79631 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44032 clock time -[SCHEDULER ITER9 LR1]: 200066 words processed Mon Nov 16 20:00:51 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.177102. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16462 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93362 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77410 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42572 clock time -[SCHEDULER ITER9 LR1]: 240045 words processed Mon Nov 16 20:01:06 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.174905. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15192 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93082 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76764 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42522 clock time -[SCHEDULER ITER9 LR1]: 280057 words processed Mon Nov 16 20:01:21 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.171743. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19048 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94746 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77299 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43339 clock time -[SCHEDULER ITER9 LR1]: 320106 words processed Mon Nov 16 20:01:36 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.169278. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24449 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95465 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80753 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43522 clock time -[SCHEDULER ITER9 LR1]: 360024 words processed Mon Nov 16 20:01:50 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.168191. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12187 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92806 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74736 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42414 clock time -[SCHEDULER ITER9 LR1]: 400089 words processed Mon Nov 16 20:02:05 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.166589. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28321 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96885 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81092 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44651 clock time -[SCHEDULER ITER9 LR1]: 440067 words processed Mon Nov 16 20:02:20 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.166948. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24516 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94979 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81487 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43194 clock time -[SCHEDULER ITER9 LR1]: 480051 words processed Mon Nov 16 20:02:35 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.167291. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27472 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95717 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82736 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43610 clock time -[SCHEDULER ITER9 LR1]: 520140 words processed Mon Nov 16 20:02:50 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.167079. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27591 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96438 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81894 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44149 clock time -[SCHEDULER ITER9 LR1]: 560132 words processed Mon Nov 16 20:03:05 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.166695. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25737 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95203 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43481 clock time -[SCHEDULER ITER9 LR1]: 600118 words processed Mon Nov 16 20:03:20 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.164905. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13899 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93008 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76359 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42472 clock time -[SCHEDULER ITER9 LR1]: 640090 words processed Mon Nov 16 20:03:35 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.163491. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14391 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93196 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76617 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42411 clock time -[SCHEDULER ITER9 LR1]: 680075 words processed Mon Nov 16 20:03:49 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.163145. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.11384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92420 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42278 clock time -[SCHEDULER ITER9 LR1]: 720043 words processed Mon Nov 16 20:04:04 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.163223. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23897 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95149 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80998 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43212 clock time -[SCHEDULER ITER9 LR1]: 760012 words processed Mon Nov 16 20:04:19 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.162981. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21179 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94529 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79619 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43125 clock time -[SCHEDULER ITER9 LR1]: 800113 words processed Mon Nov 16 20:04:33 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.163222. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16143 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94340 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76038 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43220 clock time -[SCHEDULER ITER9 LR1]: 840089 words processed Mon Nov 16 20:04:48 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.162394. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18125 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78528 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42716 clock time -[SCHEDULER ITER9 LR1]: 880052 words processed Mon Nov 16 20:05:03 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.162392. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25829 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95993 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43850 clock time +[SCHEDULER ITER9 LR1]: 40092 words processed Tue Nov 17 00:52:51 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.232103. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23223 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97236 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75796 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42776 clock time +[SCHEDULER ITER9 LR1]: 80099 words processed Tue Nov 17 00:53:06 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.236028. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18630 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95225 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74867 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41866 clock time +[SCHEDULER ITER9 LR1]: 120004 words processed Tue Nov 17 00:53:21 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.228859. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18404 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95226 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74637 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41808 clock time +[SCHEDULER ITER9 LR1]: 160114 words processed Tue Nov 17 00:53:36 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.229555. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26046 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97989 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43192 clock time +[SCHEDULER ITER9 LR1]: 200066 words processed Tue Nov 17 00:53:51 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.230362. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27221 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97545 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43056 clock time +[SCHEDULER ITER9 LR1]: 240045 words processed Tue Nov 17 00:54:06 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.226261. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32482 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98745 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79842 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43881 clock time +[SCHEDULER ITER9 LR1]: 280057 words processed Tue Nov 17 00:54:21 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.223132. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32756 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99221 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79850 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43802 clock time +[SCHEDULER ITER9 LR1]: 320106 words processed Tue Nov 17 00:54:36 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.219551. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28132 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98422 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78243 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43421 clock time +[SCHEDULER ITER9 LR1]: 360024 words processed Tue Nov 17 00:54:50 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.218183. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17014 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95399 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74229 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41762 clock time +[SCHEDULER ITER9 LR1]: 400089 words processed Tue Nov 17 00:55:05 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.216307. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23689 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97225 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76785 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43019 clock time +[SCHEDULER ITER9 LR1]: 440067 words processed Tue Nov 17 00:55:20 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.216153. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18305 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95105 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75654 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41918 clock time +[SCHEDULER ITER9 LR1]: 480051 words processed Tue Nov 17 00:55:35 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.216206. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17828 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95512 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74753 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41840 clock time +[SCHEDULER ITER9 LR1]: 520140 words processed Tue Nov 17 00:55:50 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.215684. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99580 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79133 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43741 clock time +[SCHEDULER ITER9 LR1]: 560132 words processed Tue Nov 17 00:56:05 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.215045. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24215 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96989 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77178 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42760 clock time +[SCHEDULER ITER9 LR1]: 600118 words processed Tue Nov 17 00:56:19 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.215640. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17112 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95315 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74733 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41829 clock time +[SCHEDULER ITER9 LR1]: 640090 words processed Tue Nov 17 00:56:34 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.214766. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24384 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96608 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77331 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42937 clock time +[SCHEDULER ITER9 LR1]: 680075 words processed Tue Nov 17 00:56:49 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.214549. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20459 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95893 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75999 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42143 clock time +[SCHEDULER ITER9 LR1]: 720043 words processed Tue Nov 17 00:57:04 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.214420. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17586 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95139 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75018 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41888 clock time +[SCHEDULER ITER9 LR1]: 760012 words processed Tue Nov 17 00:57:19 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.213760. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22540 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96770 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42584 clock time +[SCHEDULER ITER9 LR1]: 800113 words processed Tue Nov 17 00:57:34 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.213535. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28122 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98702 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77896 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43421 clock time +[SCHEDULER ITER9 LR1]: 840089 words processed Tue Nov 17 00:57:48 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.212476. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16595 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95177 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74353 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41802 clock time +[SCHEDULER ITER9 LR1]: 880052 words processed Tue Nov 17 00:58:02 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.212112. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17051 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74246 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41916 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER9 LR1]: Displaying result: -[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 162.33382952875> <PPL_OOV 145.38930025961> <LOGP -2010266.3742698> +[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 182.97679215081> <PPL_OOV 163.00095551912> <LOGP -2056427.6295954> [SCHEDULER ITER9 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER9 LR1]: shuffling training file ===PEEK ON TEST 9=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER9 LR1]: 40087 words processed Mon Nov 16 20:05:15 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.215174. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.71355 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79478 clock time +[SCHEDULER ITER9 LR1]: 40087 words processed Tue Nov 17 00:58:13 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.232754. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67738 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75625 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER9 LR1]: Displaying result: -[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 187.88679957257> <PPL_OOV 164.45892821205> <LOGP -182669.61606603> +[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 196.64824587583> <PPL_OOV 171.45504113653> <LOGP -184161.00583322> [SCHEDULER ITER9 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 9=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER9 LR1]: 40095 words processed Mon Nov 16 20:05:29 2015. - [SCHEDULER ITER9 LR1]: log prob per sample :-2.276313. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77046 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83022 clock time +[SCHEDULER ITER9 LR1]: 40095 words processed Tue Nov 17 00:58:26 2015. + [SCHEDULER ITER9 LR1]: log prob per sample :-2.293926. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73867 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79431 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER9 LR1]: Displaying result: -[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 193.16650117439> <PPL_OOV 173.26413664884> <LOGP -165127.15215465> +[SCHEDULER ITER9 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 202.75164379455> <PPL_OOV 181.31448315555> <LOGP -166581.98092979> [SCHEDULER ITER9 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER9 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.9... [SCHEDULER ITER10 LR1]: preparing parameters... [SCHEDULER ITER10 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.9... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 1900407 +reading chunk 1 from 46049236 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1905263 +reading chunk 2 from 46054094 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48089502 +reading chunk 3 from 92200163 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48205839 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 92316571 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94251708 +reading chunk 5 from 94220563 [SCHEDULER ITER10 LR1]: preparing parameters end. [SCHEDULER ITER10 LR1]: preparing layers... -(20:05:41 2015-11-16)[nerv] info: create layer: recurrentL1 -(20:05:41 2015-11-16)[nerv] info: create layer: sigmoidL1 -(20:05:41 2015-11-16)[nerv] info: create layer: combinerL1 -(20:05:41 2015-11-16)[nerv] info: create layer: outputL -(20:05:41 2015-11-16)[nerv] info: create layer: softmaxL -(20:05:41 2015-11-16)[nerv] info: create layer: selectL1 +(00:58:37 2015-11-17)[nerv] info: create layer: recurrentL1 +(00:58:37 2015-11-17)[nerv] info: create layer: sigmoidL1 +(00:58:37 2015-11-17)[nerv] info: create layer: combinerL1 +(00:58:37 2015-11-17)[nerv] info: create layer: outputL +(00:58:37 2015-11-17)[nerv] info: create layer: softmaxL +(00:58:37 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER10 LR1]: preparing layers end. [SCHEDULER ITER10 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -1584,193 +2076,193 @@ TNN initing storage outputL -> softmaxL ===ITERATION 10 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER10 LR1]: 40092 words processed Mon Nov 16 20:05:55 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.160846. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24036 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94857 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79273 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44562 clock time -[SCHEDULER ITER10 LR1]: 80099 words processed Mon Nov 16 20:06:09 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.160337. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18408 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92396 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78403 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42989 clock time -[SCHEDULER ITER10 LR1]: 120004 words processed Mon Nov 16 20:06:24 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.155624. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23261 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80356 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43498 clock time -[SCHEDULER ITER10 LR1]: 160114 words processed Mon Nov 16 20:06:39 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.156627. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23774 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94439 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80648 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43747 clock time -[SCHEDULER ITER10 LR1]: 200066 words processed Mon Nov 16 20:06:54 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.157317. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14825 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76768 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42792 clock time -[SCHEDULER ITER10 LR1]: 240045 words processed Mon Nov 16 20:07:09 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.155176. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33637 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95815 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85622 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44382 clock time -[SCHEDULER ITER10 LR1]: 280057 words processed Mon Nov 16 20:07:24 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.151805. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35179 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96711 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85697 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44681 clock time -[SCHEDULER ITER10 LR1]: 320106 words processed Mon Nov 16 20:07:39 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.149324. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22708 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94164 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80375 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43729 clock time -[SCHEDULER ITER10 LR1]: 360024 words processed Mon Nov 16 20:07:54 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.148109. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22268 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96914 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77386 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43788 clock time -[SCHEDULER ITER10 LR1]: 400089 words processed Mon Nov 16 20:08:09 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.146601. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26354 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97100 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80064 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43718 clock time -[SCHEDULER ITER10 LR1]: 440067 words processed Mon Nov 16 20:08:24 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.146986. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16844 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92308 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78538 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42798 clock time -[SCHEDULER ITER10 LR1]: 480051 words processed Mon Nov 16 20:08:39 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.147458. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26427 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94276 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82543 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43878 clock time -[SCHEDULER ITER10 LR1]: 520140 words processed Mon Nov 16 20:08:54 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.147224. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26497 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95469 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81114 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44236 clock time -[SCHEDULER ITER10 LR1]: 560132 words processed Mon Nov 16 20:09:09 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.147223. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18126 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92630 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79152 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42998 clock time -[SCHEDULER ITER10 LR1]: 600118 words processed Mon Nov 16 20:09:24 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.145378. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16677 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92451 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78152 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42928 clock time -[SCHEDULER ITER10 LR1]: 640090 words processed Mon Nov 16 20:09:38 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.144720. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14543 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92104 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76772 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42796 clock time -[SCHEDULER ITER10 LR1]: 680075 words processed Mon Nov 16 20:09:53 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.144278. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19397 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92990 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43395 clock time -[SCHEDULER ITER10 LR1]: 720043 words processed Mon Nov 16 20:10:08 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.144382. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22382 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93874 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80578 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43413 clock time -[SCHEDULER ITER10 LR1]: 760012 words processed Mon Nov 16 20:10:22 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.144462. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12722 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.91642 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75957 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42827 clock time -[SCHEDULER ITER10 LR1]: 800113 words processed Mon Nov 16 20:10:36 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.144791. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17292 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93579 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76966 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43720 clock time -[SCHEDULER ITER10 LR1]: 840089 words processed Mon Nov 16 20:10:51 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.144007. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22642 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94658 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79795 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43718 clock time -[SCHEDULER ITER10 LR1]: 880052 words processed Mon Nov 16 20:11:06 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.144035. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21717 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77156 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44993 clock time +[SCHEDULER ITER10 LR1]: 40092 words processed Tue Nov 17 00:58:51 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.210073. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25602 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97248 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77483 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43195 clock time +[SCHEDULER ITER10 LR1]: 80099 words processed Tue Nov 17 00:59:06 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.211256. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20154 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95320 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75918 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42145 clock time +[SCHEDULER ITER10 LR1]: 120004 words processed Tue Nov 17 00:59:21 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.203278. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23537 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96091 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77112 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42487 clock time +[SCHEDULER ITER10 LR1]: 160114 words processed Tue Nov 17 00:59:36 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.203514. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24706 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97173 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77272 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42988 clock time +[SCHEDULER ITER10 LR1]: 200066 words processed Tue Nov 17 00:59:51 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.204311. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20068 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95540 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75857 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42126 clock time +[SCHEDULER ITER10 LR1]: 240045 words processed Tue Nov 17 01:00:06 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.199779. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19267 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95280 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75869 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42050 clock time +[SCHEDULER ITER10 LR1]: 280057 words processed Tue Nov 17 01:00:21 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.196434. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23740 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97080 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76698 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42815 clock time +[SCHEDULER ITER10 LR1]: 320106 words processed Tue Nov 17 01:00:36 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.193140. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33289 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99305 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80157 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44037 clock time +[SCHEDULER ITER10 LR1]: 360024 words processed Tue Nov 17 01:00:51 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.191670. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29212 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79376 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43497 clock time +[SCHEDULER ITER10 LR1]: 400089 words processed Tue Nov 17 01:01:06 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.189883. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31550 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98916 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79849 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43819 clock time +[SCHEDULER ITER10 LR1]: 440067 words processed Tue Nov 17 01:01:21 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.189946. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34429 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98963 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44189 clock time +[SCHEDULER ITER10 LR1]: 480051 words processed Tue Nov 17 01:01:36 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.190185. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24581 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96940 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77616 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42774 clock time +[SCHEDULER ITER10 LR1]: 520140 words processed Tue Nov 17 01:01:51 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.189695. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23139 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96987 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76732 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42886 clock time +[SCHEDULER ITER10 LR1]: 560132 words processed Tue Nov 17 01:02:06 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.189184. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20477 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76170 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42155 clock time +[SCHEDULER ITER10 LR1]: 600118 words processed Tue Nov 17 01:02:21 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.187887. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20965 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96033 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76129 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42298 clock time +[SCHEDULER ITER10 LR1]: 640090 words processed Tue Nov 17 01:02:35 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.186727. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18149 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94978 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75660 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42031 clock time +[SCHEDULER ITER10 LR1]: 680075 words processed Tue Nov 17 01:02:50 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.186039. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18323 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95645 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74955 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42174 clock time +[SCHEDULER ITER10 LR1]: 720043 words processed Tue Nov 17 01:03:05 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.185979. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18323 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95402 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75166 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42092 clock time +[SCHEDULER ITER10 LR1]: 760012 words processed Tue Nov 17 01:03:20 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.185304. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33355 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98520 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80799 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43738 clock time +[SCHEDULER ITER10 LR1]: 800113 words processed Tue Nov 17 01:03:35 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.185318. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37157 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00388 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81517 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44649 clock time +[SCHEDULER ITER10 LR1]: 840089 words processed Tue Nov 17 01:03:50 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.184366. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28993 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97897 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79146 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43426 clock time +[SCHEDULER ITER10 LR1]: 880052 words processed Tue Nov 17 01:04:05 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.184212. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22044 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96287 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76545 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42368 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER10 LR1]: Displaying result: -[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 155.32157700761> <PPL_OOV 139.37671121002> <LOGP -1993215.6354609> +[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 171.10524559089> <PPL_OOV 152.86303226082> <LOGP -2030503.5489687> [SCHEDULER ITER10 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER10 LR1]: shuffling training file ===PEEK ON TEST 10=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER10 LR1]: 40087 words processed Mon Nov 16 20:11:17 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.206112. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69873 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78259 clock time +[SCHEDULER ITER10 LR1]: 40087 words processed Tue Nov 17 01:04:16 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.221232. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72207 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78148 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER10 LR1]: Displaying result: -[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 184.80096290604> <PPL_OOV 161.06280623663> <LOGP -181922.61941708> +[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 191.17046169497> <PPL_OOV 166.65930644125> <LOGP -183145.41146838> [SCHEDULER ITER10 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 10=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER10 LR1]: 40095 words processed Mon Nov 16 20:11:30 2015. - [SCHEDULER ITER10 LR1]: log prob per sample :-2.268081. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69868 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77935 clock time +[SCHEDULER ITER10 LR1]: 40095 words processed Tue Nov 17 01:04:29 2015. + [SCHEDULER ITER10 LR1]: log prob per sample :-2.283222. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73725 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79152 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER10 LR1]: Displaying result: -[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 189.96938721225> <PPL_OOV 169.83162585427> <LOGP -164486.16952174> +[SCHEDULER ITER10 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 197.52510821286> <PPL_OOV 176.60651038989> <LOGP -165739.21487712> [SCHEDULER ITER10 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER10 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.10... [SCHEDULER ITER11 LR1]: preparing parameters... [SCHEDULER ITER11 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.10... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 46044887 +reading chunk 1 from 1902876 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46049743 +reading chunk 2 from 1907733 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92246758 +reading chunk 3 from 48068859 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92363086 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +reading chunk 4 from 48185240 +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94262965 +reading chunk 5 from 94235527 [SCHEDULER ITER11 LR1]: preparing parameters end. [SCHEDULER ITER11 LR1]: preparing layers... -(20:11:41 2015-11-16)[nerv] info: create layer: recurrentL1 -(20:11:41 2015-11-16)[nerv] info: create layer: sigmoidL1 -(20:11:41 2015-11-16)[nerv] info: create layer: combinerL1 -(20:11:41 2015-11-16)[nerv] info: create layer: outputL -(20:11:41 2015-11-16)[nerv] info: create layer: softmaxL -(20:11:41 2015-11-16)[nerv] info: create layer: selectL1 +(01:04:40 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:04:40 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:04:40 2015-11-17)[nerv] info: create layer: combinerL1 +(01:04:40 2015-11-17)[nerv] info: create layer: outputL +(01:04:40 2015-11-17)[nerv] info: create layer: softmaxL +(01:04:40 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER11 LR1]: preparing layers end. [SCHEDULER ITER11 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -1798,193 +2290,193 @@ TNN initing storage outputL -> softmaxL ===ITERATION 11 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER11 LR1]: 40092 words processed Mon Nov 16 20:11:55 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.137039. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24793 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96238 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78565 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44301 clock time -[SCHEDULER ITER11 LR1]: 80099 words processed Mon Nov 16 20:12:10 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.138731. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21228 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94034 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79548 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42942 clock time -[SCHEDULER ITER11 LR1]: 120004 words processed Mon Nov 16 20:12:25 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.135105. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27656 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95557 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80949 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44057 clock time -[SCHEDULER ITER11 LR1]: 160114 words processed Mon Nov 16 20:12:40 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.137115. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28045 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80779 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44389 clock time -[SCHEDULER ITER11 LR1]: 200066 words processed Mon Nov 16 20:12:55 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.138343. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22558 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94877 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78820 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43608 clock time -[SCHEDULER ITER11 LR1]: 240045 words processed Mon Nov 16 20:13:09 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.135492. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17519 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93534 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77292 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42629 clock time -[SCHEDULER ITER11 LR1]: 280057 words processed Mon Nov 16 20:13:24 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.132313. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32819 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97650 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82577 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45015 clock time -[SCHEDULER ITER11 LR1]: 320106 words processed Mon Nov 16 20:13:39 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.130158. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34942 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97827 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83728 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45217 clock time -[SCHEDULER ITER11 LR1]: 360024 words processed Mon Nov 16 20:13:54 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.128993. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28891 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95936 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81657 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44191 clock time -[SCHEDULER ITER11 LR1]: 400089 words processed Mon Nov 16 20:14:09 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.127477. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24800 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95750 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79942 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43532 clock time -[SCHEDULER ITER11 LR1]: 440067 words processed Mon Nov 16 20:14:24 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.128128. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23082 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94445 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81337 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42869 clock time -[SCHEDULER ITER11 LR1]: 480051 words processed Mon Nov 16 20:14:39 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.128809. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24641 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95096 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80989 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43566 clock time -[SCHEDULER ITER11 LR1]: 520140 words processed Mon Nov 16 20:14:54 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.128769. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36388 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98005 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85747 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44825 clock time -[SCHEDULER ITER11 LR1]: 560132 words processed Mon Nov 16 20:15:09 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.128666. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18150 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93929 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42854 clock time -[SCHEDULER ITER11 LR1]: 600118 words processed Mon Nov 16 20:15:24 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.127160. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17267 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93180 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78464 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42593 clock time -[SCHEDULER ITER11 LR1]: 640090 words processed Mon Nov 16 20:15:39 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.126301. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16702 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93248 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77925 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42666 clock time -[SCHEDULER ITER11 LR1]: 680075 words processed Mon Nov 16 20:15:54 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.125962. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27773 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96086 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81539 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44200 clock time -[SCHEDULER ITER11 LR1]: 720043 words processed Mon Nov 16 20:16:09 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.126239. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34574 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96644 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85868 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44048 clock time -[SCHEDULER ITER11 LR1]: 760012 words processed Mon Nov 16 20:16:24 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.125843. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25474 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95045 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82331 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43228 clock time -[SCHEDULER ITER11 LR1]: 800113 words processed Mon Nov 16 20:16:39 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.126260. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37667 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98417 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86995 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44942 clock time -[SCHEDULER ITER11 LR1]: 840089 words processed Mon Nov 16 20:16:54 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.125762. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27163 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95357 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81707 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43802 clock time -[SCHEDULER ITER11 LR1]: 880052 words processed Mon Nov 16 20:17:09 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.125916. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26090 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94947 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82002 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43615 clock time +[SCHEDULER ITER11 LR1]: 40092 words processed Tue Nov 17 01:04:54 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.180225. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27306 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98276 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77184 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43577 clock time +[SCHEDULER ITER11 LR1]: 80099 words processed Tue Nov 17 01:05:09 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.179932. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25198 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96931 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77216 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42741 clock time +[SCHEDULER ITER11 LR1]: 120004 words processed Tue Nov 17 01:05:24 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.174005. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34306 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98440 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80387 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43697 clock time +[SCHEDULER ITER11 LR1]: 160114 words processed Tue Nov 17 01:05:39 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.175933. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99014 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78322 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43661 clock time +[SCHEDULER ITER11 LR1]: 200066 words processed Tue Nov 17 01:05:54 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.177108. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26994 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97541 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77833 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42911 clock time +[SCHEDULER ITER11 LR1]: 240045 words processed Tue Nov 17 01:06:09 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.173307. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25613 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97207 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77419 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42713 clock time +[SCHEDULER ITER11 LR1]: 280057 words processed Tue Nov 17 01:06:24 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.169894. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29143 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98218 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78318 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43279 clock time +[SCHEDULER ITER11 LR1]: 320106 words processed Tue Nov 17 01:06:39 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.167452. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22322 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96917 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75922 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42622 clock time +[SCHEDULER ITER11 LR1]: 360024 words processed Tue Nov 17 01:06:53 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.166278. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17076 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95298 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74091 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41901 clock time +[SCHEDULER ITER11 LR1]: 400089 words processed Tue Nov 17 01:07:08 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.164645. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22330 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97325 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75619 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42868 clock time +[SCHEDULER ITER11 LR1]: 440067 words processed Tue Nov 17 01:07:23 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.165075. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26725 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97865 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77825 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42718 clock time +[SCHEDULER ITER11 LR1]: 480051 words processed Tue Nov 17 01:07:38 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.165679. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25895 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97248 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77675 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43038 clock time +[SCHEDULER ITER11 LR1]: 520140 words processed Tue Nov 17 01:07:53 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.165441. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21843 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97101 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75450 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42695 clock time +[SCHEDULER ITER11 LR1]: 560132 words processed Tue Nov 17 01:08:07 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.165179. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17082 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95396 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74236 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41921 clock time +[SCHEDULER ITER11 LR1]: 600118 words processed Tue Nov 17 01:08:21 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.164637. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17370 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95365 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74485 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41971 clock time +[SCHEDULER ITER11 LR1]: 640090 words processed Tue Nov 17 01:08:35 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.164054. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17263 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95432 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41843 clock time +[SCHEDULER ITER11 LR1]: 680075 words processed Tue Nov 17 01:08:50 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.163403. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20557 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96264 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75704 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42345 clock time +[SCHEDULER ITER11 LR1]: 720043 words processed Tue Nov 17 01:09:05 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.163372. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97128 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77379 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43111 clock time +[SCHEDULER ITER11 LR1]: 760012 words processed Tue Nov 17 01:09:20 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.162746. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28601 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97883 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78513 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43431 clock time +[SCHEDULER ITER11 LR1]: 800113 words processed Tue Nov 17 01:09:35 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.162888. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24740 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43162 clock time +[SCHEDULER ITER11 LR1]: 840089 words processed Tue Nov 17 01:09:50 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.162197. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20326 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96051 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75540 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42431 clock time +[SCHEDULER ITER11 LR1]: 880052 words processed Tue Nov 17 01:10:05 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.162171. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18740 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95671 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74957 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42036 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER11 LR1]: Displaying result: -[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 148.71364832114> <PPL_OOV 133.680351666> <LOGP -1976369.0350324> +[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 162.26388306516> <PPL_OOV 145.30212357577> <LOGP -2010024.230432> [SCHEDULER ITER11 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER11 LR1]: shuffling training file ===PEEK ON TEST 11=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER11 LR1]: 40087 words processed Mon Nov 16 20:17:21 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.200361. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.78934 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84770 clock time +[SCHEDULER ITER11 LR1]: 40087 words processed Tue Nov 17 01:10:16 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.212312. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.74672 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79363 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER11 LR1]: Displaying result: -[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 181.46166137023> <PPL_OOV 158.99733526873> <LOGP -181460.56498772> +[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 187.44329733566> <PPL_OOV 163.39725964731> <LOGP -182437.76628055> [SCHEDULER ITER11 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 11=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER11 LR1]: 40095 words processed Mon Nov 16 20:17:34 2015. - [SCHEDULER ITER11 LR1]: log prob per sample :-2.262837. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68128 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76927 clock time +[SCHEDULER ITER11 LR1]: 40095 words processed Tue Nov 17 01:10:29 2015. + [SCHEDULER ITER11 LR1]: log prob per sample :-2.275976. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77264 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81250 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER11 LR1]: Displaying result: -[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 186.31886120305> <PPL_OOV 167.31899347048> <LOGP -164008.69715028> +[SCHEDULER ITER11 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 193.81833263359> <PPL_OOV 173.26934476521> <LOGP -165128.11503138> [SCHEDULER ITER11 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER11 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.11... [SCHEDULER ITER12 LR1]: preparing parameters... [SCHEDULER ITER12 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.11... reading chunk 0 from 0 -metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} +metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 1899123 +reading chunk 1 from 46050831 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1903979 +reading chunk 2 from 46055688 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48113072 +reading chunk 3 from 92230148 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48229392 -metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} +reading chunk 4 from 92346524 +metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94272505 +reading chunk 5 from 94248338 [SCHEDULER ITER12 LR1]: preparing parameters end. [SCHEDULER ITER12 LR1]: preparing layers... -(20:17:45 2015-11-16)[nerv] info: create layer: recurrentL1 -(20:17:45 2015-11-16)[nerv] info: create layer: sigmoidL1 -(20:17:45 2015-11-16)[nerv] info: create layer: combinerL1 -(20:17:45 2015-11-16)[nerv] info: create layer: outputL -(20:17:45 2015-11-16)[nerv] info: create layer: softmaxL -(20:17:45 2015-11-16)[nerv] info: create layer: selectL1 +(01:10:40 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:10:40 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:10:40 2015-11-17)[nerv] info: create layer: combinerL1 +(01:10:40 2015-11-17)[nerv] info: create layer: outputL +(01:10:40 2015-11-17)[nerv] info: create layer: softmaxL +(01:10:40 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER12 LR1]: preparing layers end. [SCHEDULER ITER12 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -2012,193 +2504,193 @@ TNN initing storage outputL -> softmaxL ===ITERATION 12 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER12 LR1]: 40092 words processed Mon Nov 16 20:17:59 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.123412. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31655 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96784 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82706 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45205 clock time -[SCHEDULER ITER12 LR1]: 80099 words processed Mon Nov 16 20:18:14 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.129435. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26310 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94459 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82181 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43669 clock time -[SCHEDULER ITER12 LR1]: 120004 words processed Mon Nov 16 20:18:29 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.126273. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18871 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92782 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79320 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42744 clock time -[SCHEDULER ITER12 LR1]: 160114 words processed Mon Nov 16 20:18:44 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.127267. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22501 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94613 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79576 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43822 clock time -[SCHEDULER ITER12 LR1]: 200066 words processed Mon Nov 16 20:18:59 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.127589. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27943 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94986 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83544 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43845 clock time -[SCHEDULER ITER12 LR1]: 240045 words processed Mon Nov 16 20:19:14 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.125036. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29008 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95137 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83500 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44103 clock time -[SCHEDULER ITER12 LR1]: 280057 words processed Mon Nov 16 20:19:29 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.121050. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20525 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94523 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43764 clock time -[SCHEDULER ITER12 LR1]: 320106 words processed Mon Nov 16 20:19:44 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.118508. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19000 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94195 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77888 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43634 clock time -[SCHEDULER ITER12 LR1]: 360024 words processed Mon Nov 16 20:19:59 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.117107. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22844 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44605 clock time -[SCHEDULER ITER12 LR1]: 400089 words processed Mon Nov 16 20:20:14 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.115619. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21023 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94456 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79203 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43611 clock time -[SCHEDULER ITER12 LR1]: 440067 words processed Mon Nov 16 20:20:29 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.116089. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16962 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92561 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78853 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42775 clock time -[SCHEDULER ITER12 LR1]: 480051 words processed Mon Nov 16 20:20:44 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.116697. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30042 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95413 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84571 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44143 clock time -[SCHEDULER ITER12 LR1]: 520140 words processed Mon Nov 16 20:20:59 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.116334. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34376 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97422 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83921 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45307 clock time -[SCHEDULER ITER12 LR1]: 560132 words processed Mon Nov 16 20:21:14 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.116120. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26554 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94705 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83062 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43723 clock time -[SCHEDULER ITER12 LR1]: 600118 words processed Mon Nov 16 20:21:29 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.114022. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97015 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.88220 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45057 clock time -[SCHEDULER ITER12 LR1]: 640090 words processed Mon Nov 16 20:21:44 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.112655. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17404 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92716 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78821 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42924 clock time -[SCHEDULER ITER12 LR1]: 680075 words processed Mon Nov 16 20:21:59 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.111982. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23431 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94394 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80939 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43543 clock time -[SCHEDULER ITER12 LR1]: 720043 words processed Mon Nov 16 20:22:14 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.111916. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26652 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94442 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83704 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43485 clock time -[SCHEDULER ITER12 LR1]: 760012 words processed Mon Nov 16 20:22:29 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.111100. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93516 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80718 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42949 clock time -[SCHEDULER ITER12 LR1]: 800113 words processed Mon Nov 16 20:22:44 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.111543. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28087 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95649 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83307 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44227 clock time -[SCHEDULER ITER12 LR1]: 840089 words processed Mon Nov 16 20:22:59 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.111212. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18115 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78325 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43159 clock time -[SCHEDULER ITER12 LR1]: 880052 words processed Mon Nov 16 20:23:14 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.111385. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27892 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95288 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82366 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44261 clock time +[SCHEDULER ITER12 LR1]: 40092 words processed Tue Nov 17 01:10:54 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.161136. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25635 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97397 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76151 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43395 clock time +[SCHEDULER ITER12 LR1]: 80099 words processed Tue Nov 17 01:11:09 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.159983. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19711 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95065 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74844 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42220 clock time +[SCHEDULER ITER12 LR1]: 120004 words processed Tue Nov 17 01:11:24 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.154076. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31534 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97830 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78921 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43521 clock time +[SCHEDULER ITER12 LR1]: 160114 words processed Tue Nov 17 01:11:39 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.155930. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25165 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97184 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76444 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43197 clock time +[SCHEDULER ITER12 LR1]: 200066 words processed Tue Nov 17 01:11:54 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.156832. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18373 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95233 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73984 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42281 clock time +[SCHEDULER ITER12 LR1]: 240045 words processed Tue Nov 17 01:12:09 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.153944. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31180 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97551 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79326 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43797 clock time +[SCHEDULER ITER12 LR1]: 280057 words processed Tue Nov 17 01:12:24 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.150631. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99288 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79791 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44277 clock time +[SCHEDULER ITER12 LR1]: 320106 words processed Tue Nov 17 01:12:39 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.148049. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36794 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00054 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80273 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44673 clock time +[SCHEDULER ITER12 LR1]: 360024 words processed Tue Nov 17 01:12:54 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.146847. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18953 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95181 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74686 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42379 clock time +[SCHEDULER ITER12 LR1]: 400089 words processed Tue Nov 17 01:13:09 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.145329. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24018 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96729 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76539 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43092 clock time +[SCHEDULER ITER12 LR1]: 440067 words processed Tue Nov 17 01:13:24 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.145795. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19739 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95427 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75033 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42727 clock time +[SCHEDULER ITER12 LR1]: 480051 words processed Tue Nov 17 01:13:39 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.146464. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23949 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96198 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77193 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42869 clock time +[SCHEDULER ITER12 LR1]: 520140 words processed Tue Nov 17 01:13:54 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.146077. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36956 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99841 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81109 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44511 clock time +[SCHEDULER ITER12 LR1]: 560132 words processed Tue Nov 17 01:14:09 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.145999. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21350 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95826 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75884 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42478 clock time +[SCHEDULER ITER12 LR1]: 600118 words processed Tue Nov 17 01:14:24 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.144733. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18821 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95543 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42479 clock time +[SCHEDULER ITER12 LR1]: 640090 words processed Tue Nov 17 01:14:39 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.144451. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31614 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98055 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79766 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43895 clock time +[SCHEDULER ITER12 LR1]: 680075 words processed Tue Nov 17 01:14:54 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.143942. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32660 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98161 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80122 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43805 clock time +[SCHEDULER ITER12 LR1]: 720043 words processed Tue Nov 17 01:15:09 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.143823. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24309 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76950 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43047 clock time +[SCHEDULER ITER12 LR1]: 760012 words processed Tue Nov 17 01:15:24 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.143549. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97528 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78791 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43442 clock time +[SCHEDULER ITER12 LR1]: 800113 words processed Tue Nov 17 01:15:39 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.143817. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31128 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98664 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78700 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43975 clock time +[SCHEDULER ITER12 LR1]: 840089 words processed Tue Nov 17 01:15:54 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.143030. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26105 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96700 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77695 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43188 clock time +[SCHEDULER ITER12 LR1]: 880052 words processed Tue Nov 17 01:16:09 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.143165. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28671 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97222 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78630 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43301 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER12 LR1]: Displaying result: -[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 143.61331878081> <PPL_OOV 129.28305120291> <LOGP -1962865.8381873> +[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 155.02407954573> <PPL_OOV 139.08452117388> <LOGP -1992368.3961396> [SCHEDULER ITER12 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER12 LR1]: shuffling training file ===PEEK ON TEST 12=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER12 LR1]: 40087 words processed Mon Nov 16 20:23:26 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.200053. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73371 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81521 clock time +[SCHEDULER ITER12 LR1]: 40087 words processed Tue Nov 17 01:16:20 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.205136. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.75632 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79710 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER12 LR1]: Displaying result: -[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 180.79468645492> <PPL_OOV 158.80950707114> <LOGP -181418.24970907> +[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 183.82637546451> <PPL_OOV 160.53150375807> <LOGP -181804.33337843> [SCHEDULER ITER12 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 12=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER12 LR1]: 40095 words processed Mon Nov 16 20:23:40 2015. - [SCHEDULER ITER12 LR1]: log prob per sample :-2.262970. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.80721 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86174 clock time +[SCHEDULER ITER12 LR1]: 40095 words processed Tue Nov 17 01:16:33 2015. + [SCHEDULER ITER12 LR1]: log prob per sample :-2.269139. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77463 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81002 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER12 LR1]: Displaying result: -[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 186.12441685147> <PPL_OOV 167.44182066983> <LOGP -164032.20403992> +[SCHEDULER ITER12 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 190.22401216171> <PPL_OOV 170.26138873665> <LOGP -164567.12880193> [SCHEDULER ITER12 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. -[SCHEDULER ITER12 LR1]: PPL did not improve, rejected, copying param file of last iter... +[SCHEDULER ITER12 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.12... [SCHEDULER ITER13 LR1]: preparing parameters... [SCHEDULER ITER13 LR1]: loading parameter from file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.12... reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1899123 +reading chunk 1 from 1900641 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1903979 +reading chunk 2 from 1905498 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48113072 +reading chunk 3 from 48091292 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48229392 +reading chunk 4 from 48207652 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94272505 +reading chunk 5 from 94258214 [SCHEDULER ITER13 LR1]: preparing parameters end. [SCHEDULER ITER13 LR1]: preparing layers... -(20:23:50 2015-11-16)[nerv] info: create layer: recurrentL1 -(20:23:50 2015-11-16)[nerv] info: create layer: sigmoidL1 -(20:23:50 2015-11-16)[nerv] info: create layer: combinerL1 -(20:23:50 2015-11-16)[nerv] info: create layer: outputL -(20:23:50 2015-11-16)[nerv] info: create layer: softmaxL -(20:23:50 2015-11-16)[nerv] info: create layer: selectL1 +(01:16:45 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:16:45 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:16:45 2015-11-17)[nerv] info: create layer: combinerL1 +(01:16:45 2015-11-17)[nerv] info: create layer: outputL +(01:16:45 2015-11-17)[nerv] info: create layer: softmaxL +(01:16:45 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER13 LR1]: preparing layers end. [SCHEDULER ITER13 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -2226,164 +2718,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 13 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER13 LR1]: 40092 words processed Mon Nov 16 20:24:05 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.124700. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28099 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81055 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44543 clock time -[SCHEDULER ITER13 LR1]: 80099 words processed Mon Nov 16 20:24:20 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.130431. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16269 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92640 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77226 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42814 clock time -[SCHEDULER ITER13 LR1]: 120004 words processed Mon Nov 16 20:24:35 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.126863. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24674 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93623 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81118 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43885 clock time -[SCHEDULER ITER13 LR1]: 160114 words processed Mon Nov 16 20:24:50 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.127641. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21975 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94661 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79415 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43594 clock time -[SCHEDULER ITER13 LR1]: 200066 words processed Mon Nov 16 20:25:04 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.128753. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13548 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92101 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76182 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42587 clock time -[SCHEDULER ITER13 LR1]: 240045 words processed Mon Nov 16 20:25:18 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.125405. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92411 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76064 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42991 clock time -[SCHEDULER ITER13 LR1]: 280057 words processed Mon Nov 16 20:25:33 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.121557. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26429 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95956 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80259 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44404 clock time -[SCHEDULER ITER13 LR1]: 320106 words processed Mon Nov 16 20:25:47 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.119057. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17185 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93824 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76845 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43400 clock time -[SCHEDULER ITER13 LR1]: 360024 words processed Mon Nov 16 20:26:02 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.117442. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18501 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93342 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79125 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42901 clock time -[SCHEDULER ITER13 LR1]: 400089 words processed Mon Nov 16 20:26:17 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.115856. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27431 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96218 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81362 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44619 clock time -[SCHEDULER ITER13 LR1]: 440067 words processed Mon Nov 16 20:26:32 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.116256. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19692 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93666 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78663 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43788 clock time -[SCHEDULER ITER13 LR1]: 480051 words processed Mon Nov 16 20:26:46 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.116761. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15725 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93335 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76437 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43067 clock time -[SCHEDULER ITER13 LR1]: 520140 words processed Mon Nov 16 20:27:01 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.116780. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77448 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43579 clock time -[SCHEDULER ITER13 LR1]: 560132 words processed Mon Nov 16 20:27:15 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.116520. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12441 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92225 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75614 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42675 clock time -[SCHEDULER ITER13 LR1]: 600118 words processed Mon Nov 16 20:27:29 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.114349. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.13155 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92220 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76292 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42820 clock time -[SCHEDULER ITER13 LR1]: 640090 words processed Mon Nov 16 20:27:44 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.113011. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15483 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92821 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77413 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42679 clock time -[SCHEDULER ITER13 LR1]: 680075 words processed Mon Nov 16 20:27:58 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.112316. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.12659 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92263 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76012 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42610 clock time -[SCHEDULER ITER13 LR1]: 720043 words processed Mon Nov 16 20:28:13 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.112292. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23926 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94473 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81020 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43597 clock time -[SCHEDULER ITER13 LR1]: 760012 words processed Mon Nov 16 20:28:28 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.111595. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96596 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85226 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44719 clock time -[SCHEDULER ITER13 LR1]: 800113 words processed Mon Nov 16 20:28:43 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.112014. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29536 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83048 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44474 clock time -[SCHEDULER ITER13 LR1]: 840089 words processed Mon Nov 16 20:28:58 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.111276. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92896 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78724 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42721 clock time -[SCHEDULER ITER13 LR1]: 880052 words processed Mon Nov 16 20:29:13 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.111390. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14769 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76710 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42597 clock time +[SCHEDULER ITER13 LR1]: 40092 words processed Tue Nov 17 01:16:59 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.141438. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30821 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99171 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78937 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43652 clock time +[SCHEDULER ITER13 LR1]: 80099 words processed Tue Nov 17 01:17:14 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.141106. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26486 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97179 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78006 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42582 clock time +[SCHEDULER ITER13 LR1]: 120004 words processed Tue Nov 17 01:17:29 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.135674. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23170 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96602 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76629 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42256 clock time +[SCHEDULER ITER13 LR1]: 160114 words processed Tue Nov 17 01:17:44 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.137881. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24861 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97683 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76908 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42805 clock time +[SCHEDULER ITER13 LR1]: 200066 words processed Tue Nov 17 01:17:59 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.138821. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96226 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76601 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42172 clock time +[SCHEDULER ITER13 LR1]: 240045 words processed Tue Nov 17 01:18:14 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.135555. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20617 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95997 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76050 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42052 clock time +[SCHEDULER ITER13 LR1]: 280057 words processed Tue Nov 17 01:18:29 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.132170. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26457 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98139 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77400 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43003 clock time +[SCHEDULER ITER13 LR1]: 320106 words processed Tue Nov 17 01:18:44 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.129973. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00066 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81114 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44090 clock time +[SCHEDULER ITER13 LR1]: 360024 words processed Tue Nov 17 01:18:59 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.128912. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27210 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97765 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78201 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42793 clock time +[SCHEDULER ITER13 LR1]: 400089 words processed Tue Nov 17 01:19:14 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.127346. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29508 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98633 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79151 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43303 clock time +[SCHEDULER ITER13 LR1]: 440067 words processed Tue Nov 17 01:19:29 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.128380. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26355 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97791 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78368 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42788 clock time +[SCHEDULER ITER13 LR1]: 480051 words processed Tue Nov 17 01:19:44 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.129139. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22189 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96642 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76594 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42387 clock time +[SCHEDULER ITER13 LR1]: 520140 words processed Tue Nov 17 01:19:59 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.128829. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24604 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97609 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77254 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42866 clock time +[SCHEDULER ITER13 LR1]: 560132 words processed Tue Nov 17 01:20:14 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.128876. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20458 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96145 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76079 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42055 clock time +[SCHEDULER ITER13 LR1]: 600118 words processed Tue Nov 17 01:20:29 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.127818. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29569 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98041 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79721 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43000 clock time +[SCHEDULER ITER13 LR1]: 640090 words processed Tue Nov 17 01:20:44 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.126990. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23635 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96908 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77190 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42628 clock time +[SCHEDULER ITER13 LR1]: 680075 words processed Tue Nov 17 01:20:59 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.126557. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20220 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96215 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75919 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42092 clock time +[SCHEDULER ITER13 LR1]: 720043 words processed Tue Nov 17 01:21:14 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.126524. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21439 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77135 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42087 clock time +[SCHEDULER ITER13 LR1]: 760012 words processed Tue Nov 17 01:21:29 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.125846. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28108 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98233 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43089 clock time +[SCHEDULER ITER13 LR1]: 800113 words processed Tue Nov 17 01:21:44 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.126400. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25355 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97842 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77707 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42869 clock time +[SCHEDULER ITER13 LR1]: 840089 words processed Tue Nov 17 01:21:59 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.125646. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19459 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96122 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75400 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41971 clock time +[SCHEDULER ITER13 LR1]: 880052 words processed Tue Nov 17 01:22:14 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.125781. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21180 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96329 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76221 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42102 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER13 LR1]: Displaying result: -[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 143.62007780591> <PPL_OOV 129.28060429933> <LOGP -1962858.1971088> +[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 148.6629646364> <PPL_OOV 133.62621167445> <LOGP -1976205.4988519> [SCHEDULER ITER13 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER13 LR1]: shuffling training file ===PEEK ON TEST 13=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER13 LR1]: 40087 words processed Mon Nov 16 20:29:24 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.199427. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70902 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79208 clock time +[SCHEDULER ITER13 LR1]: 40087 words processed Tue Nov 17 01:22:25 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.199339. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77549 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81756 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER13 LR1]: Displaying result: -[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 181.20405532305> <PPL_OOV 158.6828773782> <LOGP -181389.69341333> +[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 181.58984388635> <PPL_OOV 158.52809959662> <LOGP -181354.75846933> [SCHEDULER ITER13 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 13=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER13 LR1]: 40095 words processed Mon Nov 16 20:29:37 2015. - [SCHEDULER ITER13 LR1]: log prob per sample :-2.262494. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68860 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77540 clock time +[SCHEDULER ITER13 LR1]: 40095 words processed Tue Nov 17 01:22:38 2015. + [SCHEDULER ITER13 LR1]: log prob per sample :-2.265245. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.76262 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80811 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER13 LR1]: Displaying result: -[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 186.32135953827> <PPL_OOV 167.22217261963> <LOGP -163990.15523657> +[SCHEDULER ITER13 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 188.12712809794> <PPL_OOV 168.32960820323> <LOGP -164201.59940783> [SCHEDULER ITER13 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER13 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.13... @@ -2392,27 +2884,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46041450 +reading chunk 1 from 46049506 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46046306 +reading chunk 2 from 46054363 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92267138 +reading chunk 3 from 92251002 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92383441 +reading chunk 4 from 92367333 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94281630 +reading chunk 5 from 94267097 [SCHEDULER ITER14 LR1]: preparing parameters end. [SCHEDULER ITER14 LR1]: preparing layers... -(20:29:49 2015-11-16)[nerv] info: create layer: recurrentL1 -(20:29:49 2015-11-16)[nerv] info: create layer: sigmoidL1 -(20:29:49 2015-11-16)[nerv] info: create layer: combinerL1 -(20:29:49 2015-11-16)[nerv] info: create layer: outputL -(20:29:49 2015-11-16)[nerv] info: create layer: softmaxL -(20:29:49 2015-11-16)[nerv] info: create layer: selectL1 +(01:22:49 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:22:49 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:22:49 2015-11-17)[nerv] info: create layer: combinerL1 +(01:22:49 2015-11-17)[nerv] info: create layer: outputL +(01:22:49 2015-11-17)[nerv] info: create layer: softmaxL +(01:22:49 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER14 LR1]: preparing layers end. [SCHEDULER ITER14 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -2440,164 +2932,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 14 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER14 LR1]: 40092 words processed Mon Nov 16 20:30:04 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.106519. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30461 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97088 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82877 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44307 clock time -[SCHEDULER ITER14 LR1]: 80099 words processed Mon Nov 16 20:30:19 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.109678. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28269 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95321 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82989 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43324 clock time -[SCHEDULER ITER14 LR1]: 120004 words processed Mon Nov 16 20:30:34 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.106282. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30831 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95866 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84050 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43681 clock time -[SCHEDULER ITER14 LR1]: 160114 words processed Mon Nov 16 20:30:49 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.115532. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.41336 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99208 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45120 clock time -[SCHEDULER ITER14 LR1]: 200066 words processed Mon Nov 16 20:31:04 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.115626. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28454 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95420 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83684 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43619 clock time -[SCHEDULER ITER14 LR1]: 240045 words processed Mon Nov 16 20:31:19 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.114675. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21826 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94146 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80500 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43045 clock time -[SCHEDULER ITER14 LR1]: 280057 words processed Mon Nov 16 20:31:34 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.110440. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30548 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96596 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44140 clock time -[SCHEDULER ITER14 LR1]: 320106 words processed Mon Nov 16 20:31:49 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.107807. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26336 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96067 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81367 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43764 clock time -[SCHEDULER ITER14 LR1]: 360024 words processed Mon Nov 16 20:32:04 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.105960. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22322 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94159 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80968 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42895 clock time -[SCHEDULER ITER14 LR1]: 400089 words processed Mon Nov 16 20:32:19 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.104508. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26900 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95934 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82151 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43663 clock time -[SCHEDULER ITER14 LR1]: 440067 words processed Mon Nov 16 20:32:34 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.104912. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21978 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94521 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79773 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43335 clock time -[SCHEDULER ITER14 LR1]: 480051 words processed Mon Nov 16 20:32:49 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.105325. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29212 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95974 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83581 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43533 clock time -[SCHEDULER ITER14 LR1]: 520140 words processed Mon Nov 16 20:33:04 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.104824. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27934 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96269 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82311 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43878 clock time -[SCHEDULER ITER14 LR1]: 560132 words processed Mon Nov 16 20:33:19 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.104513. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26853 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95702 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81025 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43852 clock time -[SCHEDULER ITER14 LR1]: 600118 words processed Mon Nov 16 20:33:34 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.102293. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30978 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96431 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82626 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44532 clock time -[SCHEDULER ITER14 LR1]: 640090 words processed Mon Nov 16 20:33:49 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.100786. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29219 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96116 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81588 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44455 clock time -[SCHEDULER ITER14 LR1]: 680075 words processed Mon Nov 16 20:34:04 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.099982. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26772 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96005 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80443 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44164 clock time -[SCHEDULER ITER14 LR1]: 720043 words processed Mon Nov 16 20:34:19 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.099812. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19115 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93819 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78882 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42833 clock time -[SCHEDULER ITER14 LR1]: 760012 words processed Mon Nov 16 20:34:34 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.099011. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28878 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96103 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81949 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44438 clock time -[SCHEDULER ITER14 LR1]: 800113 words processed Mon Nov 16 20:34:49 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.099380. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26711 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97289 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80511 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43766 clock time -[SCHEDULER ITER14 LR1]: 840089 words processed Mon Nov 16 20:35:04 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.098603. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28103 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95753 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83225 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43645 clock time -[SCHEDULER ITER14 LR1]: 880052 words processed Mon Nov 16 20:35:18 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.098659. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92867 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42639 clock time +[SCHEDULER ITER14 LR1]: 40092 words processed Tue Nov 17 01:23:03 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.124197. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29175 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98110 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78308 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43578 clock time +[SCHEDULER ITER14 LR1]: 80099 words processed Tue Nov 17 01:23:18 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.124212. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30939 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97684 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79543 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43140 clock time +[SCHEDULER ITER14 LR1]: 120004 words processed Tue Nov 17 01:23:33 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.119187. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36242 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98666 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43756 clock time +[SCHEDULER ITER14 LR1]: 160114 words processed Tue Nov 17 01:23:48 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.121174. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33878 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98964 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80408 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43885 clock time +[SCHEDULER ITER14 LR1]: 200066 words processed Tue Nov 17 01:24:03 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.122230. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30234 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97625 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79482 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43178 clock time +[SCHEDULER ITER14 LR1]: 240045 words processed Tue Nov 17 01:24:18 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.118548. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21945 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95656 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76463 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42325 clock time +[SCHEDULER ITER14 LR1]: 280057 words processed Tue Nov 17 01:24:33 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.115349. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27424 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97620 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77865 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43268 clock time +[SCHEDULER ITER14 LR1]: 320106 words processed Tue Nov 17 01:24:48 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.113360. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.41588 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00591 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83303 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44819 clock time +[SCHEDULER ITER14 LR1]: 360024 words processed Tue Nov 17 01:25:03 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.112514. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35417 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81635 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43702 clock time +[SCHEDULER ITER14 LR1]: 400089 words processed Tue Nov 17 01:25:18 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.111125. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25575 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97303 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77398 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43068 clock time +[SCHEDULER ITER14 LR1]: 440067 words processed Tue Nov 17 01:25:33 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.111942. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20928 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95618 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76191 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42261 clock time +[SCHEDULER ITER14 LR1]: 480051 words processed Tue Nov 17 01:25:48 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.113174. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32213 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98294 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80322 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43431 clock time +[SCHEDULER ITER14 LR1]: 520140 words processed Tue Nov 17 01:26:03 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.114843. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37605 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99884 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81982 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44271 clock time +[SCHEDULER ITER14 LR1]: 560132 words processed Tue Nov 17 01:26:18 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.115302. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21912 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95760 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76524 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42399 clock time +[SCHEDULER ITER14 LR1]: 600118 words processed Tue Nov 17 01:26:33 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.114444. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20341 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95438 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75978 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42249 clock time +[SCHEDULER ITER14 LR1]: 640090 words processed Tue Nov 17 01:26:48 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.113468. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29059 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97239 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79255 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43032 clock time +[SCHEDULER ITER14 LR1]: 680075 words processed Tue Nov 17 01:27:03 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.112975. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30213 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97680 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79810 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43381 clock time +[SCHEDULER ITER14 LR1]: 720043 words processed Tue Nov 17 01:27:18 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.112843. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25668 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96743 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78023 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42790 clock time +[SCHEDULER ITER14 LR1]: 760012 words processed Tue Nov 17 01:27:33 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.112826. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22662 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95965 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76727 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42351 clock time +[SCHEDULER ITER14 LR1]: 800113 words processed Tue Nov 17 01:27:48 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.113223. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38754 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00362 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82034 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44712 clock time +[SCHEDULER ITER14 LR1]: 840089 words processed Tue Nov 17 01:28:03 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.112455. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26512 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96925 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78523 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42674 clock time +[SCHEDULER ITER14 LR1]: 880052 words processed Tue Nov 17 01:28:18 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.112529. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22167 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95943 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76519 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42479 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER14 LR1]: Displaying result: -[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 139.27471248492> <PPL_OOV 125.54833586369> <LOGP -1951031.5819951> +[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 144.01016769006> <PPL_OOV 129.605197936> <LOGP -1963870.5624736> [SCHEDULER ITER14 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER14 LR1]: shuffling training file ===PEEK ON TEST 14=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER14 LR1]: 40087 words processed Mon Nov 16 20:35:29 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.197280. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73178 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80545 clock time +[SCHEDULER ITER14 LR1]: 40087 words processed Tue Nov 17 01:28:29 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.197210. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72187 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78378 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER14 LR1]: Displaying result: -[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 179.51505756206> <PPL_OOV 157.91130158393> <LOGP -181215.20116613> +[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 180.51507944766> <PPL_OOV 157.82598259644> <LOGP -181195.85390732> [SCHEDULER ITER14 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 14=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER14 LR1]: 40095 words processed Mon Nov 16 20:35:43 2015. - [SCHEDULER ITER14 LR1]: log prob per sample :-2.260409. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77643 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83642 clock time +[SCHEDULER ITER14 LR1]: 40095 words processed Tue Nov 17 01:28:42 2015. + [SCHEDULER ITER14 LR1]: log prob per sample :-2.264146. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.76847 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81097 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER14 LR1]: Displaying result: -[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 184.73953147135> <PPL_OOV 166.37544753419> <LOGP -163827.54209865> +[SCHEDULER ITER14 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 187.16083421072> <PPL_OOV 167.65993853091> <LOGP -164073.90535959> [SCHEDULER ITER14 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER14 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.14... @@ -2606,27 +3098,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1897162 +reading chunk 1 from 1898668 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1902018 +reading chunk 2 from 1903525 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48135300 +reading chunk 3 from 48111338 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48251597 +reading chunk 4 from 48227664 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94290882 +reading chunk 5 from 94274940 [SCHEDULER ITER15 LR1]: preparing parameters end. [SCHEDULER ITER15 LR1]: preparing layers... -(20:35:55 2015-11-16)[nerv] info: create layer: recurrentL1 -(20:35:55 2015-11-16)[nerv] info: create layer: sigmoidL1 -(20:35:55 2015-11-16)[nerv] info: create layer: combinerL1 -(20:35:55 2015-11-16)[nerv] info: create layer: outputL -(20:35:55 2015-11-16)[nerv] info: create layer: softmaxL -(20:35:55 2015-11-16)[nerv] info: create layer: selectL1 +(01:28:53 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:28:53 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:28:53 2015-11-17)[nerv] info: create layer: combinerL1 +(01:28:53 2015-11-17)[nerv] info: create layer: outputL +(01:28:53 2015-11-17)[nerv] info: create layer: softmaxL +(01:28:53 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER15 LR1]: preparing layers end. [SCHEDULER ITER15 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -2654,164 +3146,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 15 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER15 LR1]: 40092 words processed Mon Nov 16 20:36:10 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.094970. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33676 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82917 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44279 clock time -[SCHEDULER ITER15 LR1]: 80099 words processed Mon Nov 16 20:36:25 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.096740. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23522 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97287 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42880 clock time -[SCHEDULER ITER15 LR1]: 120004 words processed Mon Nov 16 20:36:40 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.092736. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18877 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96327 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76281 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42617 clock time -[SCHEDULER ITER15 LR1]: 160114 words processed Mon Nov 16 20:36:55 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.096468. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28721 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99112 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80849 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43680 clock time -[SCHEDULER ITER15 LR1]: 200066 words processed Mon Nov 16 20:37:10 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.097947. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39574 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00460 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86012 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44525 clock time -[SCHEDULER ITER15 LR1]: 240045 words processed Mon Nov 16 20:37:25 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.095843. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22204 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96802 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42740 clock time -[SCHEDULER ITER15 LR1]: 280057 words processed Mon Nov 16 20:37:40 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.091781. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35603 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00636 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44904 clock time -[SCHEDULER ITER15 LR1]: 320106 words processed Mon Nov 16 20:37:55 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.089598. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32193 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00730 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80606 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44554 clock time -[SCHEDULER ITER15 LR1]: 360024 words processed Mon Nov 16 20:38:10 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.088098. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23263 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97232 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79392 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42845 clock time -[SCHEDULER ITER15 LR1]: 400089 words processed Mon Nov 16 20:38:25 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.086138. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99085 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79165 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43882 clock time -[SCHEDULER ITER15 LR1]: 440067 words processed Mon Nov 16 20:38:40 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.086957. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23976 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97762 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78951 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43257 clock time -[SCHEDULER ITER15 LR1]: 480051 words processed Mon Nov 16 20:38:55 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.087749. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28253 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98678 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81277 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43494 clock time -[SCHEDULER ITER15 LR1]: 520140 words processed Mon Nov 16 20:39:10 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.087660. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28787 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99517 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79902 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44089 clock time -[SCHEDULER ITER15 LR1]: 560132 words processed Mon Nov 16 20:39:25 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.087600. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23698 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97550 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78770 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43415 clock time -[SCHEDULER ITER15 LR1]: 600118 words processed Mon Nov 16 20:39:40 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.085650. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.40613 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00241 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87919 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44201 clock time -[SCHEDULER ITER15 LR1]: 640090 words processed Mon Nov 16 20:39:55 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.084364. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96883 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77841 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42818 clock time -[SCHEDULER ITER15 LR1]: 680075 words processed Mon Nov 16 20:40:10 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.083804. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19792 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96808 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77160 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42720 clock time -[SCHEDULER ITER15 LR1]: 720043 words processed Mon Nov 16 20:40:24 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.083820. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16935 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96247 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75649 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42640 clock time -[SCHEDULER ITER15 LR1]: 760012 words processed Mon Nov 16 20:40:39 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.083106. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18180 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96459 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76556 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42553 clock time -[SCHEDULER ITER15 LR1]: 800113 words processed Mon Nov 16 20:40:54 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.083643. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23116 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98128 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77902 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43571 clock time -[SCHEDULER ITER15 LR1]: 840089 words processed Mon Nov 16 20:41:09 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.083007. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21278 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96845 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78385 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42824 clock time -[SCHEDULER ITER15 LR1]: 880052 words processed Mon Nov 16 20:41:23 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.083226. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16957 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96438 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42622 clock time +[SCHEDULER ITER15 LR1]: 40092 words processed Tue Nov 17 01:29:07 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.116378. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29687 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98984 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78228 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43641 clock time +[SCHEDULER ITER15 LR1]: 80099 words processed Tue Nov 17 01:29:22 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.113691. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23241 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96460 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76405 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42344 clock time +[SCHEDULER ITER15 LR1]: 120004 words processed Tue Nov 17 01:29:37 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.107029. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30662 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98229 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78676 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43275 clock time +[SCHEDULER ITER15 LR1]: 160114 words processed Tue Nov 17 01:29:52 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.107861. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23667 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42841 clock time +[SCHEDULER ITER15 LR1]: 200066 words processed Tue Nov 17 01:30:07 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.108503. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18668 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95522 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74974 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41985 clock time +[SCHEDULER ITER15 LR1]: 240045 words processed Tue Nov 17 01:30:22 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.104599. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25645 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97374 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77532 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42682 clock time +[SCHEDULER ITER15 LR1]: 280057 words processed Tue Nov 17 01:30:37 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.101138. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38566 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00651 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81793 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44610 clock time +[SCHEDULER ITER15 LR1]: 320106 words processed Tue Nov 17 01:30:52 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.099228. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38333 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00619 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81888 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44747 clock time +[SCHEDULER ITER15 LR1]: 360024 words processed Tue Nov 17 01:31:07 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.098709. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19722 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95930 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42028 clock time +[SCHEDULER ITER15 LR1]: 400089 words processed Tue Nov 17 01:31:22 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.097294. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26682 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98758 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77075 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43473 clock time +[SCHEDULER ITER15 LR1]: 440067 words processed Tue Nov 17 01:31:37 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.098090. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96344 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75763 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42329 clock time +[SCHEDULER ITER15 LR1]: 480051 words processed Tue Nov 17 01:31:51 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.098889. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17460 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95710 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74305 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41899 clock time +[SCHEDULER ITER15 LR1]: 520140 words processed Tue Nov 17 01:32:06 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.099224. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34780 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99924 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80639 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43936 clock time +[SCHEDULER ITER15 LR1]: 560132 words processed Tue Nov 17 01:32:21 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.099543. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23447 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96885 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42715 clock time +[SCHEDULER ITER15 LR1]: 600118 words processed Tue Nov 17 01:32:36 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.098376. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25701 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97302 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77832 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42737 clock time +[SCHEDULER ITER15 LR1]: 640090 words processed Tue Nov 17 01:32:51 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.097524. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28187 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97725 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78689 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43108 clock time +[SCHEDULER ITER15 LR1]: 680075 words processed Tue Nov 17 01:33:06 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.097044. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26353 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97297 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78074 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43005 clock time +[SCHEDULER ITER15 LR1]: 720043 words processed Tue Nov 17 01:33:21 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.096864. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23500 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96712 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76833 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42747 clock time +[SCHEDULER ITER15 LR1]: 760012 words processed Tue Nov 17 01:33:36 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.096482. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23812 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96875 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76759 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42528 clock time +[SCHEDULER ITER15 LR1]: 800113 words processed Tue Nov 17 01:33:51 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.096947. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29705 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98864 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78726 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43673 clock time +[SCHEDULER ITER15 LR1]: 840089 words processed Tue Nov 17 01:34:06 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.096201. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21339 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96178 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42408 clock time +[SCHEDULER ITER15 LR1]: 880052 words processed Tue Nov 17 01:34:21 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.096385. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32148 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00055 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79203 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43479 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER15 LR1]: Displaying result: -[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 134.20503994822> <PPL_OOV 121.16819720447> <LOGP -1936695.1683318> +[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 138.51373531527> <PPL_OOV 124.88271450115> <LOGP -1948885.5045886> [SCHEDULER ITER15 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER15 LR1]: shuffling training file ===PEEK ON TEST 15=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER15 LR1]: 40087 words processed Mon Nov 16 20:41:34 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.191070. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69805 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78138 clock time +[SCHEDULER ITER15 LR1]: 40087 words processed Tue Nov 17 01:34:32 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.195121. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69732 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76697 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER15 LR1]: Displaying result: -[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 177.04541086517> <PPL_OOV 155.6829857727> <LOGP -180706.43817083> +[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 178.96774385229> <PPL_OOV 156.93837260582> <LOGP -180993.9534075> [SCHEDULER ITER15 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 15=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER15 LR1]: 40095 words processed Mon Nov 16 20:41:48 2015. - [SCHEDULER ITER15 LR1]: log prob per sample :-2.254691. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70130 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78411 clock time +[SCHEDULER ITER15 LR1]: 40095 words processed Tue Nov 17 01:34:45 2015. + [SCHEDULER ITER15 LR1]: log prob per sample :-2.263015. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.78230 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81915 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER15 LR1]: Displaying result: -[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 182.18941462437> <PPL_OOV 163.97982399186> <LOGP -163362.94107867> +[SCHEDULER ITER15 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 185.85640129629> <PPL_OOV 166.86234996984> <LOGP -163921.15230783> [SCHEDULER ITER15 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER15 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.15... @@ -2820,27 +3312,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46037001 +reading chunk 1 from 46045130 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46041857 +reading chunk 2 from 46049987 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92288877 +reading chunk 3 from 92270543 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92405166 +reading chunk 4 from 92386873 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94301534 +reading chunk 5 from 94284626 [SCHEDULER ITER16 LR1]: preparing parameters end. [SCHEDULER ITER16 LR1]: preparing layers... -(20:41:59 2015-11-16)[nerv] info: create layer: recurrentL1 -(20:41:59 2015-11-16)[nerv] info: create layer: sigmoidL1 -(20:41:59 2015-11-16)[nerv] info: create layer: combinerL1 -(20:41:59 2015-11-16)[nerv] info: create layer: outputL -(20:41:59 2015-11-16)[nerv] info: create layer: softmaxL -(20:41:59 2015-11-16)[nerv] info: create layer: selectL1 +(01:34:56 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:34:56 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:34:56 2015-11-17)[nerv] info: create layer: combinerL1 +(01:34:56 2015-11-17)[nerv] info: create layer: outputL +(01:34:56 2015-11-17)[nerv] info: create layer: softmaxL +(01:34:56 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER16 LR1]: preparing layers end. [SCHEDULER ITER16 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -2868,164 +3360,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 16 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER16 LR1]: 40092 words processed Mon Nov 16 20:42:14 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.083121. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30926 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96796 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83416 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44347 clock time -[SCHEDULER ITER16 LR1]: 80099 words processed Mon Nov 16 20:42:29 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.088097. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16645 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92932 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77414 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42582 clock time -[SCHEDULER ITER16 LR1]: 120004 words processed Mon Nov 16 20:42:44 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.083760. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39067 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97397 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87610 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44716 clock time -[SCHEDULER ITER16 LR1]: 160114 words processed Mon Nov 16 20:42:59 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.085542. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30781 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96763 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83081 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44361 clock time -[SCHEDULER ITER16 LR1]: 200066 words processed Mon Nov 16 20:43:14 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.087662. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21723 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93918 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80622 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42914 clock time -[SCHEDULER ITER16 LR1]: 240045 words processed Mon Nov 16 20:43:28 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.086955. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16347 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92754 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77822 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42840 clock time -[SCHEDULER ITER16 LR1]: 280057 words processed Mon Nov 16 20:43:43 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.082877. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21826 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94714 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79300 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43659 clock time -[SCHEDULER ITER16 LR1]: 320106 words processed Mon Nov 16 20:43:58 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.080652. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20718 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94749 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78940 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43507 clock time -[SCHEDULER ITER16 LR1]: 360024 words processed Mon Nov 16 20:44:13 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.078907. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15673 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77143 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42760 clock time -[SCHEDULER ITER16 LR1]: 400089 words processed Mon Nov 16 20:44:28 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.076805. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32259 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97039 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83613 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44982 clock time -[SCHEDULER ITER16 LR1]: 440067 words processed Mon Nov 16 20:44:42 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.077248. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18092 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77965 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43069 clock time -[SCHEDULER ITER16 LR1]: 480051 words processed Mon Nov 16 20:44:57 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.077975. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21850 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94431 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79569 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43543 clock time -[SCHEDULER ITER16 LR1]: 520140 words processed Mon Nov 16 20:45:12 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.077521. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24428 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95404 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80277 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44158 clock time -[SCHEDULER ITER16 LR1]: 560132 words processed Mon Nov 16 20:45:27 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.077259. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25359 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95073 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81439 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43978 clock time -[SCHEDULER ITER16 LR1]: 600118 words processed Mon Nov 16 20:45:42 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.075564. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20381 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93713 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80241 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42965 clock time -[SCHEDULER ITER16 LR1]: 640090 words processed Mon Nov 16 20:45:57 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.074164. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29047 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95805 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82692 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44345 clock time -[SCHEDULER ITER16 LR1]: 680075 words processed Mon Nov 16 20:46:12 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.073495. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19910 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93637 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79610 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43185 clock time -[SCHEDULER ITER16 LR1]: 720043 words processed Mon Nov 16 20:46:27 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.073409. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24746 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94617 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82364 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43292 clock time -[SCHEDULER ITER16 LR1]: 760012 words processed Mon Nov 16 20:46:42 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.072589. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34254 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96533 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86051 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44387 clock time -[SCHEDULER ITER16 LR1]: 800113 words processed Mon Nov 16 20:46:57 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.072909. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.41324 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98715 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.88366 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45279 clock time -[SCHEDULER ITER16 LR1]: 840089 words processed Mon Nov 16 20:47:12 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.072386. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25974 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94508 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83031 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43469 clock time -[SCHEDULER ITER16 LR1]: 880052 words processed Mon Nov 16 20:47:27 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.072623. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19708 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93603 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79750 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42940 clock time +[SCHEDULER ITER16 LR1]: 40092 words processed Tue Nov 17 01:35:10 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.097158. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26592 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97687 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77123 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43429 clock time +[SCHEDULER ITER16 LR1]: 80099 words processed Tue Nov 17 01:35:25 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.097046. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19600 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94901 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75474 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42159 clock time +[SCHEDULER ITER16 LR1]: 120004 words processed Tue Nov 17 01:35:40 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.091812. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20141 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95001 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75530 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42194 clock time +[SCHEDULER ITER16 LR1]: 160114 words processed Tue Nov 17 01:35:55 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.094211. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97909 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78649 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43467 clock time +[SCHEDULER ITER16 LR1]: 200066 words processed Tue Nov 17 01:36:10 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.094876. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24748 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96263 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77137 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42910 clock time +[SCHEDULER ITER16 LR1]: 240045 words processed Tue Nov 17 01:36:25 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.090932. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18778 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94915 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75230 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42082 clock time +[SCHEDULER ITER16 LR1]: 280057 words processed Tue Nov 17 01:36:40 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.087288. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26818 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97646 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77312 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43265 clock time +[SCHEDULER ITER16 LR1]: 320106 words processed Tue Nov 17 01:36:55 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.085495. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39843 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00162 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82801 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44858 clock time +[SCHEDULER ITER16 LR1]: 360024 words processed Tue Nov 17 01:37:10 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.085045. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35155 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98784 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81669 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43812 clock time +[SCHEDULER ITER16 LR1]: 400089 words processed Tue Nov 17 01:37:25 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.083520. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38603 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82204 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44617 clock time +[SCHEDULER ITER16 LR1]: 440067 words processed Tue Nov 17 01:37:40 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.084550. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27580 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96970 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78677 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43184 clock time +[SCHEDULER ITER16 LR1]: 480051 words processed Tue Nov 17 01:37:55 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.085379. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18268 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75072 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42199 clock time +[SCHEDULER ITER16 LR1]: 520140 words processed Tue Nov 17 01:38:10 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.085490. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23788 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96892 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76717 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43032 clock time +[SCHEDULER ITER16 LR1]: 560132 words processed Tue Nov 17 01:38:25 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.085846. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25276 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96465 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77724 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42939 clock time +[SCHEDULER ITER16 LR1]: 600118 words processed Tue Nov 17 01:38:40 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.084108. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19835 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95171 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75885 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42307 clock time +[SCHEDULER ITER16 LR1]: 640090 words processed Tue Nov 17 01:38:55 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.083068. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18706 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94996 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75458 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42169 clock time +[SCHEDULER ITER16 LR1]: 680075 words processed Tue Nov 17 01:39:10 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.082583. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21738 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96030 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76486 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42502 clock time +[SCHEDULER ITER16 LR1]: 720043 words processed Tue Nov 17 01:39:25 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.082472. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25232 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96734 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77507 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42963 clock time +[SCHEDULER ITER16 LR1]: 760012 words processed Tue Nov 17 01:39:40 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.082694. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18634 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94966 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75347 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42204 clock time +[SCHEDULER ITER16 LR1]: 800113 words processed Tue Nov 17 01:39:55 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.083364. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22996 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96807 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76177 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43085 clock time +[SCHEDULER ITER16 LR1]: 840089 words processed Tue Nov 17 01:40:10 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.082735. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23876 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96152 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77479 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42862 clock time +[SCHEDULER ITER16 LR1]: 880052 words processed Tue Nov 17 01:40:25 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.082996. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29314 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97277 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79299 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43445 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER16 LR1]: Displaying result: -[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 130.82155308403> <PPL_OOV 118.24002290514> <LOGP -1926819.0836068> +[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 134.13029533507> <PPL_OOV 121.08994838557> <LOGP -1936434.3700653> [SCHEDULER ITER16 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER16 LR1]: shuffling training file ===PEEK ON TEST 16=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER16 LR1]: 40087 words processed Mon Nov 16 20:47:38 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.187912. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70385 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79068 clock time +[SCHEDULER ITER16 LR1]: 40087 words processed Tue Nov 17 01:40:36 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.191237. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.71964 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78416 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER16 LR1]: Displaying result: -[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 176.00780059298> <PPL_OOV 154.44074852616> <LOGP -180419.64323953> +[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 177.45967549374> <PPL_OOV 155.48519677698> <LOGP -180660.92819474> [SCHEDULER ITER16 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 16=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER16 LR1]: 40095 words processed Mon Nov 16 20:47:51 2015. - [SCHEDULER ITER16 LR1]: log prob per sample :-2.251572. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.71377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79789 clock time +[SCHEDULER ITER16 LR1]: 40095 words processed Tue Nov 17 01:40:49 2015. + [SCHEDULER ITER16 LR1]: log prob per sample :-2.259248. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.74678 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79713 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER16 LR1]: Displaying result: -[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 181.17022255751> <PPL_OOV 162.74880131209> <LOGP -163121.55282167> +[SCHEDULER ITER16 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 184.59397762463> <PPL_OOV 165.63585620884> <LOGP -163684.82537337> [SCHEDULER ITER16 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER16 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.16... @@ -3034,27 +3526,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1895307 +reading chunk 1 from 1896914 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1900163 +reading chunk 2 from 1901771 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48162572 +reading chunk 3 from 48135609 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48278837 +reading chunk 4 from 48251953 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94312867 +reading chunk 5 from 94294577 [SCHEDULER ITER17 LR1]: preparing parameters end. [SCHEDULER ITER17 LR1]: preparing layers... -(20:48:03 2015-11-16)[nerv] info: create layer: recurrentL1 -(20:48:03 2015-11-16)[nerv] info: create layer: sigmoidL1 -(20:48:03 2015-11-16)[nerv] info: create layer: combinerL1 -(20:48:03 2015-11-16)[nerv] info: create layer: outputL -(20:48:03 2015-11-16)[nerv] info: create layer: softmaxL -(20:48:03 2015-11-16)[nerv] info: create layer: selectL1 +(01:41:00 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:41:00 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:41:00 2015-11-17)[nerv] info: create layer: combinerL1 +(01:41:00 2015-11-17)[nerv] info: create layer: outputL +(01:41:00 2015-11-17)[nerv] info: create layer: softmaxL +(01:41:00 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER17 LR1]: preparing layers end. [SCHEDULER ITER17 LR1]: Generate and initing TNN ... <input> selectL1 0 @@ -3082,164 +3574,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 17 LR 1.000000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER17 LR1]: 40092 words processed Mon Nov 16 20:48:17 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.075734. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27238 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98829 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79487 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43812 clock time -[SCHEDULER ITER17 LR1]: 80099 words processed Mon Nov 16 20:48:32 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.074042. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24662 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97035 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79780 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42866 clock time -[SCHEDULER ITER17 LR1]: 120004 words processed Mon Nov 16 20:48:47 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.069547. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34370 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99026 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84207 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43849 clock time -[SCHEDULER ITER17 LR1]: 160114 words processed Mon Nov 16 20:49:02 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.073792. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37477 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01261 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82775 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45199 clock time -[SCHEDULER ITER17 LR1]: 200066 words processed Mon Nov 16 20:49:16 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.073678. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18163 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96440 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75995 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42618 clock time -[SCHEDULER ITER17 LR1]: 240045 words processed Mon Nov 16 20:49:30 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.071819. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18333 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96070 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76624 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42578 clock time -[SCHEDULER ITER17 LR1]: 280057 words processed Mon Nov 16 20:49:45 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.067970. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98351 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79003 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43534 clock time -[SCHEDULER ITER17 LR1]: 320106 words processed Mon Nov 16 20:50:00 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.066148. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34355 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00283 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83681 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44330 clock time -[SCHEDULER ITER17 LR1]: 360024 words processed Mon Nov 16 20:50:15 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.064714. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23665 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97473 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78616 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43134 clock time -[SCHEDULER ITER17 LR1]: 400089 words processed Mon Nov 16 20:50:30 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.062772. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25798 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98781 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79072 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43747 clock time -[SCHEDULER ITER17 LR1]: 440067 words processed Mon Nov 16 20:50:45 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.063663. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23909 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97682 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78809 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43342 clock time -[SCHEDULER ITER17 LR1]: 480051 words processed Mon Nov 16 20:51:00 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.064452. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20330 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96739 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77672 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42808 clock time -[SCHEDULER ITER17 LR1]: 520140 words processed Mon Nov 16 20:51:15 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.064445. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37985 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01586 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83501 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45224 clock time -[SCHEDULER ITER17 LR1]: 560132 words processed Mon Nov 16 20:51:30 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.064322. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27297 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98557 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79801 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43799 clock time -[SCHEDULER ITER17 LR1]: 600118 words processed Mon Nov 16 20:51:45 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.062340. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33929 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99930 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82752 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44481 clock time -[SCHEDULER ITER17 LR1]: 640090 words processed Mon Nov 16 20:52:00 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.061531. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26143 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98059 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79720 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43557 clock time -[SCHEDULER ITER17 LR1]: 680075 words processed Mon Nov 16 20:52:15 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.060991. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30209 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99344 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80840 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44144 clock time -[SCHEDULER ITER17 LR1]: 720043 words processed Mon Nov 16 20:52:30 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.060957. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33596 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99923 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82105 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44480 clock time -[SCHEDULER ITER17 LR1]: 760012 words processed Mon Nov 16 20:52:45 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.060003. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27826 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98675 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79913 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43780 clock time -[SCHEDULER ITER17 LR1]: 800113 words processed Mon Nov 16 20:53:00 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.060353. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34891 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00902 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81952 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45237 clock time -[SCHEDULER ITER17 LR1]: 840089 words processed Mon Nov 16 20:53:15 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.060411. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22108 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97408 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77594 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43088 clock time -[SCHEDULER ITER17 LR1]: 880052 words processed Mon Nov 16 20:53:30 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.060802. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30403 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99124 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80942 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44023 clock time +[SCHEDULER ITER17 LR1]: 40092 words processed Tue Nov 17 01:41:14 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.082835. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32355 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99335 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79692 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43866 clock time +[SCHEDULER ITER17 LR1]: 80099 words processed Tue Nov 17 01:41:29 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.082799. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22563 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96150 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42116 clock time +[SCHEDULER ITER17 LR1]: 120004 words processed Tue Nov 17 01:41:44 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.078698. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23092 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96252 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42243 clock time +[SCHEDULER ITER17 LR1]: 160114 words processed Tue Nov 17 01:41:59 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.080389. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26723 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98172 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77510 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43113 clock time +[SCHEDULER ITER17 LR1]: 200066 words processed Tue Nov 17 01:42:14 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.081359. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18895 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95581 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75287 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41778 clock time +[SCHEDULER ITER17 LR1]: 240045 words processed Tue Nov 17 01:42:29 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.077826. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18760 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95483 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75171 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41860 clock time +[SCHEDULER ITER17 LR1]: 280057 words processed Tue Nov 17 01:42:44 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.074372. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23323 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97443 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76017 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42721 clock time +[SCHEDULER ITER17 LR1]: 320106 words processed Tue Nov 17 01:42:59 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.072589. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22972 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97339 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76090 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42745 clock time +[SCHEDULER ITER17 LR1]: 360024 words processed Tue Nov 17 01:43:14 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.072108. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32857 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98888 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80723 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43602 clock time +[SCHEDULER ITER17 LR1]: 400089 words processed Tue Nov 17 01:43:29 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.070747. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36026 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00091 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81374 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43998 clock time +[SCHEDULER ITER17 LR1]: 440067 words processed Tue Nov 17 01:43:44 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.071871. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17679 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95643 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74602 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41893 clock time +[SCHEDULER ITER17 LR1]: 480051 words processed Tue Nov 17 01:43:59 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.072731. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17181 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95449 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74530 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41870 clock time +[SCHEDULER ITER17 LR1]: 520140 words processed Tue Nov 17 01:44:14 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.073258. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26406 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98174 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77554 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43266 clock time +[SCHEDULER ITER17 LR1]: 560132 words processed Tue Nov 17 01:44:29 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.073656. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24712 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96970 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77855 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42623 clock time +[SCHEDULER ITER17 LR1]: 600118 words processed Tue Nov 17 01:44:44 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.072206. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97652 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78212 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42776 clock time +[SCHEDULER ITER17 LR1]: 640090 words processed Tue Nov 17 01:44:59 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.071177. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17906 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95537 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75109 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41897 clock time +[SCHEDULER ITER17 LR1]: 680075 words processed Tue Nov 17 01:45:14 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.070671. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18134 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95529 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74991 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41954 clock time +[SCHEDULER ITER17 LR1]: 720043 words processed Tue Nov 17 01:45:29 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.070539. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31403 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98772 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80072 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43333 clock time +[SCHEDULER ITER17 LR1]: 760012 words processed Tue Nov 17 01:45:44 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.069850. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30152 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98078 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79870 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43032 clock time +[SCHEDULER ITER17 LR1]: 800113 words processed Tue Nov 17 01:45:59 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.070369. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35853 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00193 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81232 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44252 clock time +[SCHEDULER ITER17 LR1]: 840089 words processed Tue Nov 17 01:46:14 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.069971. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24223 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96842 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77499 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42577 clock time +[SCHEDULER ITER17 LR1]: 880052 words processed Tue Nov 17 01:46:29 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.070321. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18666 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95526 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75521 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41889 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER17 LR1]: Displaying result: -[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 127.15444419997> <PPL_OOV 115.06947267974> <LOGP -1915845.8596768> +[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 130.10568904217> <PPL_OOV 117.6122627072> <LOGP -1924669.9669223> [SCHEDULER ITER17 LR1]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER17 LR1]: shuffling training file ===PEEK ON TEST 17=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER17 LR1]: 40087 words processed Mon Nov 16 20:53:41 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.184861. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69809 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77810 clock time +[SCHEDULER ITER17 LR1]: 40087 words processed Tue Nov 17 01:46:40 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.191054. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.75496 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80089 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER17 LR1]: Displaying result: -[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 175.32009789052> <PPL_OOV 153.46425910208> <LOGP -180192.57737275> +[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 177.55797219602> <PPL_OOV 155.34470306374> <LOGP -180628.56631505> [SCHEDULER ITER17 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 17=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER17 LR1]: 40095 words processed Mon Nov 16 20:53:54 2015. - [SCHEDULER ITER17 LR1]: log prob per sample :-2.249517. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.80166 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84378 clock time +[SCHEDULER ITER17 LR1]: 40095 words processed Tue Nov 17 01:46:53 2015. + [SCHEDULER ITER17 LR1]: log prob per sample :-2.259630. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68302 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75741 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER17 LR1]: Displaying result: -[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 180.31979996826> <PPL_OOV 161.69079166391> <LOGP -162912.62679993> +[SCHEDULER ITER17 LR1]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 184.6258385289> <PPL_OOV 165.43985811395> <LOGP -163646.89738658> [SCHEDULER ITER17 LR1]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER17 LR1]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.17... @@ -3248,27 +3740,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46031439 +reading chunk 1 from 46040379 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46036295 +reading chunk 2 from 46045236 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92315356 +reading chunk 3 from 92292549 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92431616 +reading chunk 4 from 92408883 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94325933 +reading chunk 5 from 94305051 [SCHEDULER ITER18 LR0.6]: preparing parameters end. [SCHEDULER ITER18 LR0.6]: preparing layers... -(20:54:05 2015-11-16)[nerv] info: create layer: recurrentL1 -(20:54:05 2015-11-16)[nerv] info: create layer: sigmoidL1 -(20:54:05 2015-11-16)[nerv] info: create layer: combinerL1 -(20:54:05 2015-11-16)[nerv] info: create layer: outputL -(20:54:05 2015-11-16)[nerv] info: create layer: softmaxL -(20:54:05 2015-11-16)[nerv] info: create layer: selectL1 +(01:47:04 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:47:04 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:47:04 2015-11-17)[nerv] info: create layer: combinerL1 +(01:47:04 2015-11-17)[nerv] info: create layer: outputL +(01:47:04 2015-11-17)[nerv] info: create layer: softmaxL +(01:47:04 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER18 LR0.6]: preparing layers end. [SCHEDULER ITER18 LR0.6]: Generate and initing TNN ... <input> selectL1 0 @@ -3296,164 +3788,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 18 LR 0.600000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER18 LR0.6]: 40092 words processed Mon Nov 16 20:54:19 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.034218. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25745 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80661 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44200 clock time -[SCHEDULER ITER18 LR0.6]: 80099 words processed Mon Nov 16 20:54:34 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.031870. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31327 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95763 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84274 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43935 clock time -[SCHEDULER ITER18 LR0.6]: 120004 words processed Mon Nov 16 20:54:49 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.025285. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19025 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93110 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78589 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42993 clock time -[SCHEDULER ITER18 LR0.6]: 160114 words processed Mon Nov 16 20:55:04 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.025948. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26754 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95652 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81728 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43860 clock time -[SCHEDULER ITER18 LR0.6]: 200066 words processed Mon Nov 16 20:55:19 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.024900. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26959 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96784 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79937 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44687 clock time -[SCHEDULER ITER18 LR0.6]: 240045 words processed Mon Nov 16 20:55:34 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.021138. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26777 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97406 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79689 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43863 clock time -[SCHEDULER ITER18 LR0.6]: 280057 words processed Mon Nov 16 20:55:49 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.016731. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21745 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94592 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78846 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43716 clock time -[SCHEDULER ITER18 LR0.6]: 320106 words processed Mon Nov 16 20:56:04 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.014350. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.44575 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99327 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.89758 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45644 clock time -[SCHEDULER ITER18 LR0.6]: 360024 words processed Mon Nov 16 20:56:19 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.012541. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95430 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84021 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43770 clock time -[SCHEDULER ITER18 LR0.6]: 400089 words processed Mon Nov 16 20:56:34 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.009680. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19987 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94475 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77959 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43726 clock time -[SCHEDULER ITER18 LR0.6]: 440067 words processed Mon Nov 16 20:56:49 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.010105. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33034 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96528 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83960 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44646 clock time -[SCHEDULER ITER18 LR0.6]: 480051 words processed Mon Nov 16 20:57:04 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.010581. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26811 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95434 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81236 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44047 clock time -[SCHEDULER ITER18 LR0.6]: 520140 words processed Mon Nov 16 20:57:19 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.010123. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19916 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94413 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78156 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43676 clock time -[SCHEDULER ITER18 LR0.6]: 560132 words processed Mon Nov 16 20:57:34 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.009508. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16646 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92927 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77893 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42786 clock time -[SCHEDULER ITER18 LR0.6]: 600118 words processed Mon Nov 16 20:57:49 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.007130. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28794 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95431 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83818 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43863 clock time -[SCHEDULER ITER18 LR0.6]: 640090 words processed Mon Nov 16 20:58:04 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.005205. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24691 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94252 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43309 clock time -[SCHEDULER ITER18 LR0.6]: 680075 words processed Mon Nov 16 20:58:19 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.004151. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30513 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95557 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84603 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43733 clock time -[SCHEDULER ITER18 LR0.6]: 720043 words processed Mon Nov 16 20:58:34 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.003549. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24535 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94291 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82343 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43094 clock time -[SCHEDULER ITER18 LR0.6]: 760012 words processed Mon Nov 16 20:58:49 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.002349. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33777 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96807 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84990 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44530 clock time -[SCHEDULER ITER18 LR0.6]: 800113 words processed Mon Nov 16 20:59:04 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.002397. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28491 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96010 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83108 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44137 clock time -[SCHEDULER ITER18 LR0.6]: 840089 words processed Mon Nov 16 20:59:19 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.001577. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26981 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94970 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43478 clock time -[SCHEDULER ITER18 LR0.6]: 880052 words processed Mon Nov 16 20:59:34 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.001526. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26841 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94925 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83012 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43490 clock time +[SCHEDULER ITER18 LR0.6]: 40092 words processed Tue Nov 17 01:47:18 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.043667. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27354 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97688 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78487 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43053 clock time +[SCHEDULER ITER18 LR0.6]: 80099 words processed Tue Nov 17 01:47:33 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.039896. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29561 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97514 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79646 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43033 clock time +[SCHEDULER ITER18 LR0.6]: 120004 words processed Tue Nov 17 01:47:48 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.033742. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96295 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77775 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42551 clock time +[SCHEDULER ITER18 LR0.6]: 160114 words processed Tue Nov 17 01:48:03 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.035984. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33994 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99450 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80942 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43759 clock time +[SCHEDULER ITER18 LR0.6]: 200066 words processed Tue Nov 17 01:48:18 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.035035. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35697 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99148 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81827 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43689 clock time +[SCHEDULER ITER18 LR0.6]: 240045 words processed Tue Nov 17 01:48:33 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.030310. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29005 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97492 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79727 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42971 clock time +[SCHEDULER ITER18 LR0.6]: 280057 words processed Tue Nov 17 01:48:48 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.025772. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29545 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98404 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78987 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43357 clock time +[SCHEDULER ITER18 LR0.6]: 320106 words processed Tue Nov 17 01:49:03 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.023427. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39999 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00644 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83127 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44537 clock time +[SCHEDULER ITER18 LR0.6]: 360024 words processed Tue Nov 17 01:49:18 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.021796. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21554 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95796 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76826 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42206 clock time +[SCHEDULER ITER18 LR0.6]: 400089 words processed Tue Nov 17 01:49:33 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.019102. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25584 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97352 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78116 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42966 clock time +[SCHEDULER ITER18 LR0.6]: 440067 words processed Tue Nov 17 01:49:48 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.019677. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21961 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77078 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42320 clock time +[SCHEDULER ITER18 LR0.6]: 480051 words processed Tue Nov 17 01:50:03 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.020225. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27855 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97398 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79211 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42937 clock time +[SCHEDULER ITER18 LR0.6]: 520140 words processed Tue Nov 17 01:50:18 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.019858. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77939 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43047 clock time +[SCHEDULER ITER18 LR0.6]: 560132 words processed Tue Nov 17 01:50:33 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.019758. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27310 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97260 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79087 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42955 clock time +[SCHEDULER ITER18 LR0.6]: 600118 words processed Tue Nov 17 01:50:48 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.017512. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22239 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96167 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77086 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42284 clock time +[SCHEDULER ITER18 LR0.6]: 640090 words processed Tue Nov 17 01:51:03 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.015774. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21496 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77075 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42161 clock time +[SCHEDULER ITER18 LR0.6]: 680075 words processed Tue Nov 17 01:51:18 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.014732. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96868 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78621 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43066 clock time +[SCHEDULER ITER18 LR0.6]: 720043 words processed Tue Nov 17 01:51:33 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.014130. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20750 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95739 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76617 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42080 clock time +[SCHEDULER ITER18 LR0.6]: 760012 words processed Tue Nov 17 01:51:48 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.012778. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95703 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76828 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42143 clock time +[SCHEDULER ITER18 LR0.6]: 800113 words processed Tue Nov 17 01:52:03 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.012906. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25861 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97590 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78034 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43002 clock time +[SCHEDULER ITER18 LR0.6]: 840089 words processed Tue Nov 17 01:52:18 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.012001. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20671 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95733 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76595 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42122 clock time +[SCHEDULER ITER18 LR0.6]: 880052 words processed Tue Nov 17 01:52:33 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.012052. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21005 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95728 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76664 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42153 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER18 LR0.6]: Displaying result: -[SCHEDULER ITER18 LR0.6]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 110.94944312459> <PPL_OOV 100.36982641987> <LOGP -1860668.2920595> +[SCHEDULER ITER18 LR0.6]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 113.75890428527> <PPL_OOV 102.8317079383> <LOGP -1870451.1714354> [SCHEDULER ITER18 LR0.6]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER18 LR0.6]: shuffling training file ===PEEK ON TEST 18=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER18 LR0.6]: 40087 words processed Mon Nov 16 20:59:46 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.163278. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.84783 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.88318 clock time +[SCHEDULER ITER18 LR0.6]: 40087 words processed Tue Nov 17 01:52:44 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.159080. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73520 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78969 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER18 LR0.6]: Displaying result: -[SCHEDULER ITER18 LR0.6]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 166.16167676808> <PPL_OOV 146.04731194544> <LOGP -178419.20481298> +[SCHEDULER ITER18 LR0.6]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 164.81314830342> <PPL_OOV 144.38164096479> <LOGP -178008.57197272> [SCHEDULER ITER18 LR0.6]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 18=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER18 LR0.6]: 40095 words processed Mon Nov 16 21:00:00 2015. - [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.230708. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77264 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83802 clock time +[SCHEDULER ITER18 LR0.6]: 40095 words processed Tue Nov 17 01:52:57 2015. + [SCHEDULER ITER18 LR0.6]: log prob per sample :-2.229502. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.71650 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77727 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER18 LR0.6]: Displaying result: -[SCHEDULER ITER18 LR0.6]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 170.55259243863> <PPL_OOV 153.42154597608> <LOGP -161230.97752031> +[SCHEDULER ITER18 LR0.6]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 171.0439239354> <PPL_OOV 153.42799000947> <LOGP -161232.32297019> [SCHEDULER ITER18 LR0.6]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER18 LR0.6]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.18... @@ -3462,27 +3954,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1892928 +reading chunk 1 from 1895256 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1897784 +reading chunk 2 from 1900112 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48183711 +reading chunk 3 from 48153092 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48299967 +reading chunk 4 from 48269401 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94329634 +reading chunk 5 from 94307755 [SCHEDULER ITER19 LR0.36]: preparing parameters end. [SCHEDULER ITER19 LR0.36]: preparing layers... -(21:00:12 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:00:12 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:00:12 2015-11-16)[nerv] info: create layer: combinerL1 -(21:00:12 2015-11-16)[nerv] info: create layer: outputL -(21:00:12 2015-11-16)[nerv] info: create layer: softmaxL -(21:00:12 2015-11-16)[nerv] info: create layer: selectL1 +(01:53:08 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:53:08 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:53:08 2015-11-17)[nerv] info: create layer: combinerL1 +(01:53:08 2015-11-17)[nerv] info: create layer: outputL +(01:53:08 2015-11-17)[nerv] info: create layer: softmaxL +(01:53:08 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER19 LR0.36]: preparing layers end. [SCHEDULER ITER19 LR0.36]: Generate and initing TNN ... <input> selectL1 0 @@ -3510,164 +4002,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 19 LR 0.360000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER19 LR0.36]: 40092 words processed Mon Nov 16 21:00:27 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.997014. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37699 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01214 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83925 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44752 clock time -[SCHEDULER ITER19 LR0.36]: 80099 words processed Mon Nov 16 21:00:42 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.994733. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22590 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97446 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77180 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42930 clock time -[SCHEDULER ITER19 LR0.36]: 120004 words processed Mon Nov 16 21:00:57 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.988085. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25920 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97689 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78876 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43313 clock time -[SCHEDULER ITER19 LR0.36]: 160114 words processed Mon Nov 16 21:01:12 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.988949. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28391 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99172 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79733 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43870 clock time -[SCHEDULER ITER19 LR0.36]: 200066 words processed Mon Nov 16 21:01:27 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.987750. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23944 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97421 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78807 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42972 clock time -[SCHEDULER ITER19 LR0.36]: 240045 words processed Mon Nov 16 21:01:42 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.983959. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27650 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98485 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79756 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43655 clock time -[SCHEDULER ITER19 LR0.36]: 280057 words processed Mon Nov 16 21:01:57 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.979621. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32363 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00628 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44894 clock time -[SCHEDULER ITER19 LR0.36]: 320106 words processed Mon Nov 16 21:02:12 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.977246. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30365 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00369 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79604 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44530 clock time -[SCHEDULER ITER19 LR0.36]: 360024 words processed Mon Nov 16 21:02:27 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.975444. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26700 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98481 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79095 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43630 clock time -[SCHEDULER ITER19 LR0.36]: 400089 words processed Mon Nov 16 21:02:42 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.972430. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24528 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98750 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77773 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43695 clock time -[SCHEDULER ITER19 LR0.36]: 440067 words processed Mon Nov 16 21:02:57 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.972825. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25762 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97900 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79388 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43285 clock time -[SCHEDULER ITER19 LR0.36]: 480051 words processed Mon Nov 16 21:03:12 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.973155. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39802 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01075 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85441 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44901 clock time -[SCHEDULER ITER19 LR0.36]: 520140 words processed Mon Nov 16 21:03:27 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.972554. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27265 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99593 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78666 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44004 clock time -[SCHEDULER ITER19 LR0.36]: 560132 words processed Mon Nov 16 21:03:42 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.971809. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20244 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97249 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76500 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43149 clock time -[SCHEDULER ITER19 LR0.36]: 600118 words processed Mon Nov 16 21:03:56 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.969291. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17425 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96787 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75004 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42625 clock time -[SCHEDULER ITER19 LR0.36]: 640090 words processed Mon Nov 16 21:04:11 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.967195. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19563 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96958 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76280 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42895 clock time -[SCHEDULER ITER19 LR0.36]: 680075 words processed Mon Nov 16 21:04:26 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.965907. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17501 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96561 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75222 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42697 clock time -[SCHEDULER ITER19 LR0.36]: 720043 words processed Mon Nov 16 21:04:41 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.964968. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19445 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97205 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75933 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42914 clock time -[SCHEDULER ITER19 LR0.36]: 760012 words processed Mon Nov 16 21:04:56 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.963337. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29222 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99264 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79571 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44209 clock time -[SCHEDULER ITER19 LR0.36]: 800113 words processed Mon Nov 16 21:05:11 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.963206. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24557 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98569 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78092 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43640 clock time -[SCHEDULER ITER19 LR0.36]: 840089 words processed Mon Nov 16 21:05:26 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.962238. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26359 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97833 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80461 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43177 clock time -[SCHEDULER ITER19 LR0.36]: 880052 words processed Mon Nov 16 21:05:41 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.961986. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28655 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98580 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80948 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43586 clock time +[SCHEDULER ITER19 LR0.36]: 40092 words processed Tue Nov 17 01:53:22 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-2.007280. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24917 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94650 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78681 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43190 clock time +[SCHEDULER ITER19 LR0.36]: 80099 words processed Tue Nov 17 01:53:37 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-2.003431. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21244 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92585 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78045 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42475 clock time +[SCHEDULER ITER19 LR0.36]: 120004 words processed Tue Nov 17 01:53:52 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.997580. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17392 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92030 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76025 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41897 clock time +[SCHEDULER ITER19 LR0.36]: 160114 words processed Tue Nov 17 01:54:07 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.998525. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35687 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96740 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82842 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44737 clock time +[SCHEDULER ITER19 LR0.36]: 200066 words processed Tue Nov 17 01:54:22 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.997707. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20081 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92793 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77514 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42417 clock time +[SCHEDULER ITER19 LR0.36]: 240045 words processed Tue Nov 17 01:54:37 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.993389. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20717 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92648 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77940 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42406 clock time +[SCHEDULER ITER19 LR0.36]: 280057 words processed Tue Nov 17 01:54:52 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.989080. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30914 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95728 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81173 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43671 clock time +[SCHEDULER ITER19 LR0.36]: 320106 words processed Tue Nov 17 01:55:07 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.986892. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27853 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95212 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79993 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43404 clock time +[SCHEDULER ITER19 LR0.36]: 360024 words processed Tue Nov 17 01:55:22 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.985177. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22903 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92929 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79252 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42595 clock time +[SCHEDULER ITER19 LR0.36]: 400089 words processed Tue Nov 17 01:55:37 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.982339. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27249 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94919 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43509 clock time +[SCHEDULER ITER19 LR0.36]: 440067 words processed Tue Nov 17 01:55:52 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.982908. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19287 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92585 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77800 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42229 clock time +[SCHEDULER ITER19 LR0.36]: 480051 words processed Tue Nov 17 01:56:07 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.983385. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18713 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92615 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77466 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42188 clock time +[SCHEDULER ITER19 LR0.36]: 520140 words processed Tue Nov 17 01:56:22 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.982902. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96292 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81588 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44336 clock time +[SCHEDULER ITER19 LR0.36]: 560132 words processed Tue Nov 17 01:56:36 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.982661. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15741 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.91929 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76034 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41779 clock time +[SCHEDULER ITER19 LR0.36]: 600118 words processed Tue Nov 17 01:56:51 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.980337. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21376 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93380 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77952 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42540 clock time +[SCHEDULER ITER19 LR0.36]: 640090 words processed Tue Nov 17 01:57:06 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.978348. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21490 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92927 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78240 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42527 clock time +[SCHEDULER ITER19 LR0.36]: 680075 words processed Tue Nov 17 01:57:21 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.977145. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16478 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.92048 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76536 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42010 clock time +[SCHEDULER ITER19 LR0.36]: 720043 words processed Tue Nov 17 01:57:36 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.976308. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16876 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.91711 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76918 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41994 clock time +[SCHEDULER ITER19 LR0.36]: 760012 words processed Tue Nov 17 01:57:50 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.974690. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16215 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.91665 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76501 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41846 clock time +[SCHEDULER ITER19 LR0.36]: 800113 words processed Tue Nov 17 01:58:05 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.974618. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26550 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94873 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79863 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43380 clock time +[SCHEDULER ITER19 LR0.36]: 840089 words processed Tue Nov 17 01:58:20 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.973580. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23765 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93495 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79226 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43029 clock time +[SCHEDULER ITER19 LR0.36]: 880052 words processed Tue Nov 17 01:58:35 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-1.973415. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24271 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.93573 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79528 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43002 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER19 LR0.36]: Displaying result: -[SCHEDULER ITER19 LR0.36]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 101.11776818014> <PPL_OOV 91.627273951145> <LOGP -1823876.640057> +[SCHEDULER ITER19 LR0.36]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 103.89160588136> <PPL_OOV 94.068763218622> <LOGP -1834493.1676572> [SCHEDULER ITER19 LR0.36]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER19 LR0.36]: shuffling training file ===PEEK ON TEST 19=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER19 LR0.36]: 40087 words processed Mon Nov 16 21:05:53 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-2.145398. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.80456 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84858 clock time +[SCHEDULER ITER19 LR0.36]: 40087 words processed Tue Nov 17 01:58:46 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-2.136208. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77459 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82211 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER19 LR0.36]: Displaying result: -[SCHEDULER ITER19 LR0.36]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 159.60093901295> <PPL_OOV 140.12256441497> <LOGP -176936.66077397> +[SCHEDULER ITER19 LR0.36]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 157.02937166312> <PPL_OOV 137.23737609848> <LOGP -176191.85042634> [SCHEDULER ITER19 LR0.36]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 19=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER19 LR0.36]: 40095 words processed Mon Nov 16 21:06:06 2015. - [SCHEDULER ITER19 LR0.36]: log prob per sample :-2.214779. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70569 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78041 clock time +[SCHEDULER ITER19 LR0.36]: 40095 words processed Tue Nov 17 01:58:59 2015. + [SCHEDULER ITER19 LR0.36]: log prob per sample :-2.208516. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.82932 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85088 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER19 LR0.36]: Displaying result: -[SCHEDULER ITER19 LR0.36]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 163.71076230308> <PPL_OOV 147.1341216716> <LOGP -159890.54046095> +[SCHEDULER ITER19 LR0.36]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 162.67946310829> <PPL_OOV 145.6455470188> <LOGP -159564.80228371> [SCHEDULER ITER19 LR0.36]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER19 LR0.36]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.19... @@ -3676,27 +4168,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46028677 +reading chunk 1 from 46036780 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46033533 +reading chunk 2 from 46041636 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92322350 +reading chunk 3 from 92297326 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92438569 +reading chunk 4 from 92413618 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94330677 +reading chunk 5 from 94308206 [SCHEDULER ITER20 LR0.216]: preparing parameters end. [SCHEDULER ITER20 LR0.216]: preparing layers... -(21:06:17 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:06:17 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:06:17 2015-11-16)[nerv] info: create layer: combinerL1 -(21:06:17 2015-11-16)[nerv] info: create layer: outputL -(21:06:17 2015-11-16)[nerv] info: create layer: softmaxL -(21:06:17 2015-11-16)[nerv] info: create layer: selectL1 +(01:59:10 2015-11-17)[nerv] info: create layer: recurrentL1 +(01:59:10 2015-11-17)[nerv] info: create layer: sigmoidL1 +(01:59:10 2015-11-17)[nerv] info: create layer: combinerL1 +(01:59:10 2015-11-17)[nerv] info: create layer: outputL +(01:59:10 2015-11-17)[nerv] info: create layer: softmaxL +(01:59:10 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER20 LR0.216]: preparing layers end. [SCHEDULER ITER20 LR0.216]: Generate and initing TNN ... <input> selectL1 0 @@ -3724,164 +4216,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 20 LR 0.216000=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER20 LR0.216]: 40092 words processed Mon Nov 16 21:06:31 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.971407. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22962 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94759 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79420 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43827 clock time -[SCHEDULER ITER20 LR0.216]: 80099 words processed Mon Nov 16 21:06:45 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.968297. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17542 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93152 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77037 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43042 clock time -[SCHEDULER ITER20 LR0.216]: 120004 words processed Mon Nov 16 21:07:00 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.961757. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20230 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93354 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78631 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43216 clock time -[SCHEDULER ITER20 LR0.216]: 160114 words processed Mon Nov 16 21:07:15 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.962674. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23618 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94806 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79761 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43949 clock time -[SCHEDULER ITER20 LR0.216]: 200066 words processed Mon Nov 16 21:07:30 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.961535. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21250 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93132 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80796 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42903 clock time -[SCHEDULER ITER20 LR0.216]: 240045 words processed Mon Nov 16 21:07:45 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.957763. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19982 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92943 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80018 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42891 clock time -[SCHEDULER ITER20 LR0.216]: 280057 words processed Mon Nov 16 21:08:00 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.953523. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26136 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95105 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81946 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43855 clock time -[SCHEDULER ITER20 LR0.216]: 320106 words processed Mon Nov 16 21:08:15 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.951224. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22527 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95061 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79067 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44015 clock time -[SCHEDULER ITER20 LR0.216]: 360024 words processed Mon Nov 16 21:08:30 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.949400. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31292 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96120 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82623 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44692 clock time -[SCHEDULER ITER20 LR0.216]: 400089 words processed Mon Nov 16 21:08:45 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.946355. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36310 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97948 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45700 clock time -[SCHEDULER ITER20 LR0.216]: 440067 words processed Mon Nov 16 21:09:00 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.946745. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26369 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94909 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81182 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44092 clock time -[SCHEDULER ITER20 LR0.216]: 480051 words processed Mon Nov 16 21:09:15 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.947007. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29088 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95335 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82959 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44103 clock time -[SCHEDULER ITER20 LR0.216]: 520140 words processed Mon Nov 16 21:09:30 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.946379. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26552 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82563 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43845 clock time -[SCHEDULER ITER20 LR0.216]: 560132 words processed Mon Nov 16 21:09:45 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.945623. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22786 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93792 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81791 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42946 clock time -[SCHEDULER ITER20 LR0.216]: 600118 words processed Mon Nov 16 21:10:00 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.943088. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25843 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94355 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82783 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43338 clock time -[SCHEDULER ITER20 LR0.216]: 640090 words processed Mon Nov 16 21:10:15 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.940800. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94591 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83241 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43544 clock time -[SCHEDULER ITER20 LR0.216]: 680075 words processed Mon Nov 16 21:10:30 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.939424. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20027 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93066 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80238 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43030 clock time -[SCHEDULER ITER20 LR0.216]: 720043 words processed Mon Nov 16 21:10:45 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.938325. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19832 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80032 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42895 clock time -[SCHEDULER ITER20 LR0.216]: 760012 words processed Mon Nov 16 21:11:00 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.936527. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18817 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93531 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78682 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42979 clock time -[SCHEDULER ITER20 LR0.216]: 800113 words processed Mon Nov 16 21:11:15 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.936323. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22231 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94653 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79684 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43774 clock time -[SCHEDULER ITER20 LR0.216]: 840089 words processed Mon Nov 16 21:11:30 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.935251. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19464 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93182 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79775 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43002 clock time -[SCHEDULER ITER20 LR0.216]: 880052 words processed Mon Nov 16 21:11:45 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.934865. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23933 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94265 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81283 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43417 clock time +[SCHEDULER ITER20 LR0.216]: 40092 words processed Tue Nov 17 01:59:24 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.981529. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28691 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99532 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76925 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43463 clock time +[SCHEDULER ITER20 LR0.216]: 80099 words processed Tue Nov 17 01:59:39 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.977850. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21482 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96620 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75015 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42248 clock time +[SCHEDULER ITER20 LR0.216]: 120004 words processed Tue Nov 17 01:59:54 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.972350. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21997 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96758 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75141 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42335 clock time +[SCHEDULER ITER20 LR0.216]: 160114 words processed Tue Nov 17 02:00:09 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.973418. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30381 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99799 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77883 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43410 clock time +[SCHEDULER ITER20 LR0.216]: 200066 words processed Tue Nov 17 02:00:24 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.972559. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30315 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98677 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78837 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43394 clock time +[SCHEDULER ITER20 LR0.216]: 240045 words processed Tue Nov 17 02:00:39 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.968260. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33679 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99763 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79782 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43777 clock time +[SCHEDULER ITER20 LR0.216]: 280057 words processed Tue Nov 17 02:00:54 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.964016. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30520 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99875 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77879 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43726 clock time +[SCHEDULER ITER20 LR0.216]: 320106 words processed Tue Nov 17 02:01:09 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.961876. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28592 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99072 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77412 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43399 clock time +[SCHEDULER ITER20 LR0.216]: 360024 words processed Tue Nov 17 02:01:24 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.960100. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33161 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99328 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79845 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43734 clock time +[SCHEDULER ITER20 LR0.216]: 400089 words processed Tue Nov 17 02:01:39 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.957160. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38948 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.01426 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81705 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44663 clock time +[SCHEDULER ITER20 LR0.216]: 440067 words processed Tue Nov 17 02:01:54 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.957730. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99067 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79164 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43601 clock time +[SCHEDULER ITER20 LR0.216]: 480051 words processed Tue Nov 17 02:02:09 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.958163. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19547 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96511 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74672 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42200 clock time +[SCHEDULER ITER20 LR0.216]: 520140 words processed Tue Nov 17 02:02:24 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.957670. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23894 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98029 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75848 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42907 clock time +[SCHEDULER ITER20 LR0.216]: 560132 words processed Tue Nov 17 02:02:39 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.957347. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18878 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96217 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74765 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42079 clock time +[SCHEDULER ITER20 LR0.216]: 600118 words processed Tue Nov 17 02:02:54 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.954947. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18871 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96244 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74575 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42066 clock time +[SCHEDULER ITER20 LR0.216]: 640090 words processed Tue Nov 17 02:03:09 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.952798. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22587 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97209 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75660 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42420 clock time +[SCHEDULER ITER20 LR0.216]: 680075 words processed Tue Nov 17 02:03:24 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.951496. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18990 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96305 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74651 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42099 clock time +[SCHEDULER ITER20 LR0.216]: 720043 words processed Tue Nov 17 02:03:39 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.950527. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19142 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96296 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74616 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42103 clock time +[SCHEDULER ITER20 LR0.216]: 760012 words processed Tue Nov 17 02:03:54 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.948769. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19267 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96259 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74640 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42083 clock time +[SCHEDULER ITER20 LR0.216]: 800113 words processed Tue Nov 17 02:04:09 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.948570. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25073 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98325 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76370 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43084 clock time +[SCHEDULER ITER20 LR0.216]: 840089 words processed Tue Nov 17 02:04:24 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.947455. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25431 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76760 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42910 clock time +[SCHEDULER ITER20 LR0.216]: 880052 words processed Tue Nov 17 02:04:39 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-1.947175. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23859 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76380 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42552 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER20 LR0.216]: Displaying result: -[SCHEDULER ITER20 LR0.216]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 94.858763770931> <PPL_OOV 86.075078953778> <LOGP -1798640.7750352> +[SCHEDULER ITER20 LR0.216]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 97.662329395387> <PPL_OOV 88.548497236> <LOGP -1810078.2200094> [SCHEDULER ITER20 LR0.216]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER20 LR0.216]: shuffling training file ===PEEK ON TEST 20=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER20 LR0.216]: 40087 words processed Mon Nov 16 21:11:57 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-2.131186. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.75590 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82386 clock time +[SCHEDULER ITER20 LR0.216]: 40087 words processed Tue Nov 17 02:04:50 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-2.122304. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73690 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78848 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER20 LR0.216]: Displaying result: -[SCHEDULER ITER20 LR0.216]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 154.88722747527> <PPL_OOV 135.61386955113> <LOGP -175765.82763284> +[SCHEDULER ITER20 LR0.216]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 152.57073335601> <PPL_OOV 133.09751975169> <LOGP -175095.33001305> [SCHEDULER ITER20 LR0.216]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 20=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER20 LR0.216]: 40095 words processed Mon Nov 16 21:12:11 2015. - [SCHEDULER ITER20 LR0.216]: log prob per sample :-2.202088. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.74235 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81553 clock time +[SCHEDULER ITER20 LR0.216]: 40095 words processed Tue Nov 17 02:05:03 2015. + [SCHEDULER ITER20 LR0.216]: log prob per sample :-2.195560. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.76514 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80811 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER20 LR0.216]: Displaying result: -[SCHEDULER ITER20 LR0.216]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 158.87666734941> <PPL_OOV 142.46254184851> <LOGP -158856.96281008> +[SCHEDULER ITER20 LR0.216]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 157.83094717853> <PPL_OOV 141.11133347799> <LOGP -158551.68544082> [SCHEDULER ITER20 LR0.216]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER20 LR0.216]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.20... @@ -3890,27 +4382,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1891422 +reading chunk 1 from 1894050 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1896278 +reading chunk 2 from 1898903 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48186065 +reading chunk 3 from 48155663 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48302281 +reading chunk 4 from 48271943 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94330600 +reading chunk 5 from 94307817 [SCHEDULER ITER21 LR0.1296]: preparing parameters end. [SCHEDULER ITER21 LR0.1296]: preparing layers... -(21:12:22 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:12:22 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:12:22 2015-11-16)[nerv] info: create layer: combinerL1 -(21:12:22 2015-11-16)[nerv] info: create layer: outputL -(21:12:22 2015-11-16)[nerv] info: create layer: softmaxL -(21:12:22 2015-11-16)[nerv] info: create layer: selectL1 +(02:05:15 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:05:15 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:05:15 2015-11-17)[nerv] info: create layer: combinerL1 +(02:05:15 2015-11-17)[nerv] info: create layer: outputL +(02:05:15 2015-11-17)[nerv] info: create layer: softmaxL +(02:05:15 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER21 LR0.1296]: preparing layers end. [SCHEDULER ITER21 LR0.1296]: Generate and initing TNN ... <input> selectL1 0 @@ -3938,164 +4430,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 21 LR 0.129600=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER21 LR0.1296]: 40092 words processed Mon Nov 16 21:12:37 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.954227. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34756 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00467 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81957 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44675 clock time -[SCHEDULER ITER21 LR0.1296]: 80099 words processed Mon Nov 16 21:12:52 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.950706. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27143 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98442 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43554 clock time -[SCHEDULER ITER21 LR0.1296]: 120004 words processed Mon Nov 16 21:13:07 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.944022. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20431 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96757 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76389 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42821 clock time -[SCHEDULER ITER21 LR0.1296]: 160114 words processed Mon Nov 16 21:13:22 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.944942. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23465 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98528 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76677 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43513 clock time -[SCHEDULER ITER21 LR0.1296]: 200066 words processed Mon Nov 16 21:13:37 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.943798. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18373 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96675 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75395 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42719 clock time -[SCHEDULER ITER21 LR0.1296]: 240045 words processed Mon Nov 16 21:13:52 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.939964. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25409 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97913 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79033 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43266 clock time -[SCHEDULER ITER21 LR0.1296]: 280057 words processed Mon Nov 16 21:14:07 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.935737. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27576 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99447 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78164 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43937 clock time -[SCHEDULER ITER21 LR0.1296]: 320106 words processed Mon Nov 16 21:14:22 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.933468. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35426 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01180 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81053 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45017 clock time -[SCHEDULER ITER21 LR0.1296]: 360024 words processed Mon Nov 16 21:14:37 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.931656. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97378 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77168 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43218 clock time -[SCHEDULER ITER21 LR0.1296]: 400089 words processed Mon Nov 16 21:14:52 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.928579. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31833 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00297 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80294 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44639 clock time -[SCHEDULER ITER21 LR0.1296]: 440067 words processed Mon Nov 16 21:15:07 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.928992. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24081 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77876 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43484 clock time -[SCHEDULER ITER21 LR0.1296]: 480051 words processed Mon Nov 16 21:15:22 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.929176. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18611 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96436 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76267 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42682 clock time -[SCHEDULER ITER21 LR0.1296]: 520140 words processed Mon Nov 16 21:15:37 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.928482. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23616 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98585 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77081 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43720 clock time -[SCHEDULER ITER21 LR0.1296]: 560132 words processed Mon Nov 16 21:15:52 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.927679. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20601 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96832 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77303 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42811 clock time -[SCHEDULER ITER21 LR0.1296]: 600118 words processed Mon Nov 16 21:16:07 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.925138. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20391 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97032 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77028 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42722 clock time -[SCHEDULER ITER21 LR0.1296]: 640090 words processed Mon Nov 16 21:16:22 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.922796. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24838 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98380 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78104 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43485 clock time -[SCHEDULER ITER21 LR0.1296]: 680075 words processed Mon Nov 16 21:16:36 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.921362. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16326 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96331 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74818 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42500 clock time -[SCHEDULER ITER21 LR0.1296]: 720043 words processed Mon Nov 16 21:16:51 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.920138. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20506 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96921 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77065 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42829 clock time -[SCHEDULER ITER21 LR0.1296]: 760012 words processed Mon Nov 16 21:17:06 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.918226. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25844 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97623 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80184 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42974 clock time -[SCHEDULER ITER21 LR0.1296]: 800113 words processed Mon Nov 16 21:17:21 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.917948. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25590 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98707 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78588 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43778 clock time -[SCHEDULER ITER21 LR0.1296]: 840089 words processed Mon Nov 16 21:17:36 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.916767. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27524 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97933 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80926 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43237 clock time -[SCHEDULER ITER21 LR0.1296]: 880052 words processed Mon Nov 16 21:17:51 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.916267. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20955 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97582 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76386 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43003 clock time +[SCHEDULER ITER21 LR0.1296]: 40092 words processed Tue Nov 17 02:05:29 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.964006. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31852 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99210 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79216 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43812 clock time +[SCHEDULER ITER21 LR0.1296]: 80099 words processed Tue Nov 17 02:05:44 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.960321. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27241 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97162 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78438 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42698 clock time +[SCHEDULER ITER21 LR0.1296]: 120004 words processed Tue Nov 17 02:05:59 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.955006. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36320 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99130 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81563 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43577 clock time +[SCHEDULER ITER21 LR0.1296]: 160114 words processed Tue Nov 17 02:06:14 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.956131. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98947 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79081 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43373 clock time +[SCHEDULER ITER21 LR0.1296]: 200066 words processed Tue Nov 17 02:06:29 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.955298. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22627 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96308 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76749 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42164 clock time +[SCHEDULER ITER21 LR0.1296]: 240045 words processed Tue Nov 17 02:06:44 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.950983. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34363 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98768 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43492 clock time +[SCHEDULER ITER21 LR0.1296]: 280057 words processed Tue Nov 17 02:06:59 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.946828. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00091 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81438 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44236 clock time +[SCHEDULER ITER21 LR0.1296]: 320106 words processed Tue Nov 17 02:07:14 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.944750. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25980 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97838 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77524 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42912 clock time +[SCHEDULER ITER21 LR0.1296]: 360024 words processed Tue Nov 17 02:07:29 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.942985. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20577 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95810 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76111 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42013 clock time +[SCHEDULER ITER21 LR0.1296]: 400089 words processed Tue Nov 17 02:07:44 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.939985. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25783 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97766 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77471 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42947 clock time +[SCHEDULER ITER21 LR0.1296]: 440067 words processed Tue Nov 17 02:07:59 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.940516. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29150 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97876 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79280 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42982 clock time +[SCHEDULER ITER21 LR0.1296]: 480051 words processed Tue Nov 17 02:08:14 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.940943. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25575 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97091 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78318 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42515 clock time +[SCHEDULER ITER21 LR0.1296]: 520140 words processed Tue Nov 17 02:08:29 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.940430. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.40419 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82802 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44610 clock time +[SCHEDULER ITER21 LR0.1296]: 560132 words processed Tue Nov 17 02:08:44 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.940051. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22578 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96273 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77152 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42252 clock time +[SCHEDULER ITER21 LR0.1296]: 600118 words processed Tue Nov 17 02:08:59 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.937604. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21226 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96103 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76526 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42026 clock time +[SCHEDULER ITER21 LR0.1296]: 640090 words processed Tue Nov 17 02:09:14 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.935357. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28755 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97674 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79395 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42858 clock time +[SCHEDULER ITER21 LR0.1296]: 680075 words processed Tue Nov 17 02:09:29 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.933969. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27127 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97090 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78966 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42814 clock time +[SCHEDULER ITER21 LR0.1296]: 720043 words processed Tue Nov 17 02:09:44 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.932894. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31589 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98040 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80393 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43303 clock time +[SCHEDULER ITER21 LR0.1296]: 760012 words processed Tue Nov 17 02:09:59 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.931031. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35174 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98948 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81271 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43822 clock time +[SCHEDULER ITER21 LR0.1296]: 800113 words processed Tue Nov 17 02:10:14 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.930732. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37880 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00124 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82550 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44480 clock time +[SCHEDULER ITER21 LR0.1296]: 840089 words processed Tue Nov 17 02:10:29 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.929552. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23206 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96250 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77321 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42299 clock time +[SCHEDULER ITER21 LR0.1296]: 880052 words processed Tue Nov 17 02:10:44 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-1.929191. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26589 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97208 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78358 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42827 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER21 LR0.1296]: Displaying result: -[SCHEDULER ITER21 LR0.1296]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 90.78547262985> <PPL_OOV 82.462162737312> <LOGP -1781329.3097165> +[SCHEDULER ITER21 LR0.1296]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 93.606357535878> <PPL_OOV 84.952640858682> <LOGP -1793341.6104206> [SCHEDULER ITER21 LR0.1296]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER21 LR0.1296]: shuffling training file ===PEEK ON TEST 21=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER21 LR0.1296]: 40087 words processed Mon Nov 16 21:18:03 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-2.121687. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.74755 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81581 clock time +[SCHEDULER ITER21 LR0.1296]: 40087 words processed Tue Nov 17 02:10:55 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-2.114255. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.71811 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78452 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER21 LR0.1296]: Displaying result: -[SCHEDULER ITER21 LR0.1296]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 152.04549072345> <PPL_OOV 132.75281765476> <LOGP -175002.49612076> +[SCHEDULER ITER21 LR0.1296]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 150.07142046608> <PPL_OOV 130.76844701191> <LOGP -174463.33949945> [SCHEDULER ITER21 LR0.1296]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 21=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER21 LR0.1296]: 40095 words processed Mon Nov 16 21:18:17 2015. - [SCHEDULER ITER21 LR0.1296]: log prob per sample :-2.193534. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.79080 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83936 clock time +[SCHEDULER ITER21 LR0.1296]: 40095 words processed Tue Nov 17 02:11:08 2015. + [SCHEDULER ITER21 LR0.1296]: log prob per sample :-2.187961. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.75940 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81100 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER21 LR0.1296]: Displaying result: -[SCHEDULER ITER21 LR0.1296]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 155.85681868536> <PPL_OOV 139.4274434935> <LOGP -158167.12797675> +[SCHEDULER ITER21 LR0.1296]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 155.11279588996> <PPL_OOV 138.55671587422> <LOGP -157966.45037284> [SCHEDULER ITER21 LR0.1296]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER21 LR0.1296]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.21... @@ -4104,27 +4596,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46028104 +reading chunk 1 from 46035256 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46032959 +reading chunk 2 from 46040108 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92322760 +reading chunk 3 from 92296841 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92438976 +reading chunk 4 from 92413108 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94329928 +reading chunk 5 from 94306801 [SCHEDULER ITER22 LR0.07776]: preparing parameters end. [SCHEDULER ITER22 LR0.07776]: preparing layers... -(21:18:29 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:18:29 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:18:29 2015-11-16)[nerv] info: create layer: combinerL1 -(21:18:29 2015-11-16)[nerv] info: create layer: outputL -(21:18:29 2015-11-16)[nerv] info: create layer: softmaxL -(21:18:29 2015-11-16)[nerv] info: create layer: selectL1 +(02:11:20 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:11:20 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:11:20 2015-11-17)[nerv] info: create layer: combinerL1 +(02:11:20 2015-11-17)[nerv] info: create layer: outputL +(02:11:20 2015-11-17)[nerv] info: create layer: softmaxL +(02:11:20 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER22 LR0.07776]: preparing layers end. [SCHEDULER ITER22 LR0.07776]: Generate and initing TNN ... <input> selectL1 0 @@ -4152,164 +4644,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 22 LR 0.077760=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER22 LR0.07776]: 40092 words processed Mon Nov 16 21:18:43 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.941979. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26998 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95849 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80843 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44462 clock time -[SCHEDULER ITER22 LR0.07776]: 80099 words processed Mon Nov 16 21:18:58 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.938250. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24674 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94153 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81465 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43493 clock time -[SCHEDULER ITER22 LR0.07776]: 120004 words processed Mon Nov 16 21:19:12 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.931563. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17270 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92859 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77460 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42850 clock time -[SCHEDULER ITER22 LR0.07776]: 160114 words processed Mon Nov 16 21:19:27 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.932540. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32325 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97095 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82620 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45205 clock time -[SCHEDULER ITER22 LR0.07776]: 200066 words processed Mon Nov 16 21:19:42 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.931450. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22387 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94297 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79414 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43599 clock time -[SCHEDULER ITER22 LR0.07776]: 240045 words processed Mon Nov 16 21:19:57 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.927571. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25087 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94717 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80457 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44078 clock time -[SCHEDULER ITER22 LR0.07776]: 280057 words processed Mon Nov 16 21:20:12 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.923315. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28819 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96121 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82465 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44073 clock time -[SCHEDULER ITER22 LR0.07776]: 320106 words processed Mon Nov 16 21:20:27 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.921082. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28432 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96028 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82644 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44158 clock time -[SCHEDULER ITER22 LR0.07776]: 360024 words processed Mon Nov 16 21:20:42 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.919259. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18644 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93340 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78701 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42970 clock time -[SCHEDULER ITER22 LR0.07776]: 400089 words processed Mon Nov 16 21:20:57 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.916177. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.42563 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99012 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.88633 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45547 clock time -[SCHEDULER ITER22 LR0.07776]: 440067 words processed Mon Nov 16 21:21:12 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.916615. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94346 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81526 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43426 clock time -[SCHEDULER ITER22 LR0.07776]: 480051 words processed Mon Nov 16 21:21:27 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.916756. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30678 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95800 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84641 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43868 clock time -[SCHEDULER ITER22 LR0.07776]: 520140 words processed Mon Nov 16 21:21:42 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.916061. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20650 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94582 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78724 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43664 clock time -[SCHEDULER ITER22 LR0.07776]: 560132 words processed Mon Nov 16 21:21:57 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.915241. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20045 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93540 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80025 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42992 clock time -[SCHEDULER ITER22 LR0.07776]: 600118 words processed Mon Nov 16 21:22:12 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.912721. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36492 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97455 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86344 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44664 clock time -[SCHEDULER ITER22 LR0.07776]: 640090 words processed Mon Nov 16 21:22:27 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.910376. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19528 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93380 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79227 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43230 clock time -[SCHEDULER ITER22 LR0.07776]: 680075 words processed Mon Nov 16 21:22:42 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.908921. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18505 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93262 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78906 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43041 clock time -[SCHEDULER ITER22 LR0.07776]: 720043 words processed Mon Nov 16 21:22:57 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.907630. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95004 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83331 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43541 clock time -[SCHEDULER ITER22 LR0.07776]: 760012 words processed Mon Nov 16 21:23:12 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.905650. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29667 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95535 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84302 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43935 clock time -[SCHEDULER ITER22 LR0.07776]: 800113 words processed Mon Nov 16 21:23:27 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.905321. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26400 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95929 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81147 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44161 clock time -[SCHEDULER ITER22 LR0.07776]: 840089 words processed Mon Nov 16 21:23:42 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.904079. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26143 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95320 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81042 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44227 clock time -[SCHEDULER ITER22 LR0.07776]: 880052 words processed Mon Nov 16 21:23:57 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.903519. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19051 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93774 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78244 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43316 clock time +[SCHEDULER ITER22 LR0.07776]: 40092 words processed Tue Nov 17 02:11:34 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.951876. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29870 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99807 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78163 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43327 clock time +[SCHEDULER ITER22 LR0.07776]: 80099 words processed Tue Nov 17 02:11:49 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.948115. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21909 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96681 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42062 clock time +[SCHEDULER ITER22 LR0.07776]: 120004 words processed Tue Nov 17 02:12:04 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.942954. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23511 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97294 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76418 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42206 clock time +[SCHEDULER ITER22 LR0.07776]: 160114 words processed Tue Nov 17 02:12:19 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.944119. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28767 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99055 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78040 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43118 clock time +[SCHEDULER ITER22 LR0.07776]: 200066 words processed Tue Nov 17 02:12:34 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.943320. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21568 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96950 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75895 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42136 clock time +[SCHEDULER ITER22 LR0.07776]: 240045 words processed Tue Nov 17 02:12:49 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.939028. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22671 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97103 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76509 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42256 clock time +[SCHEDULER ITER22 LR0.07776]: 280057 words processed Tue Nov 17 02:13:04 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.934927. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26174 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98496 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77160 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42949 clock time +[SCHEDULER ITER22 LR0.07776]: 320106 words processed Tue Nov 17 02:13:19 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.932901. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98544 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77093 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42962 clock time +[SCHEDULER ITER22 LR0.07776]: 360024 words processed Tue Nov 17 02:13:34 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.931143. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25269 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97950 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42670 clock time +[SCHEDULER ITER22 LR0.07776]: 400089 words processed Tue Nov 17 02:13:49 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.928088. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38195 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.01329 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82095 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44362 clock time +[SCHEDULER ITER22 LR0.07776]: 440067 words processed Tue Nov 17 02:14:04 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.928607. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20245 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96824 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75345 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42015 clock time +[SCHEDULER ITER22 LR0.07776]: 480051 words processed Tue Nov 17 02:14:19 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.929036. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19903 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96763 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75467 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42070 clock time +[SCHEDULER ITER22 LR0.07776]: 520140 words processed Tue Nov 17 02:14:34 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.928505. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27332 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98879 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78030 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43229 clock time +[SCHEDULER ITER22 LR0.07776]: 560132 words processed Tue Nov 17 02:14:49 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.928065. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99533 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80584 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43596 clock time +[SCHEDULER ITER22 LR0.07776]: 600118 words processed Tue Nov 17 02:15:04 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.925599. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27555 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98504 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78555 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42991 clock time +[SCHEDULER ITER22 LR0.07776]: 640090 words processed Tue Nov 17 02:15:19 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.923311. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19726 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96571 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75467 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41973 clock time +[SCHEDULER ITER22 LR0.07776]: 680075 words processed Tue Nov 17 02:15:34 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.921873. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26284 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97904 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78386 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42711 clock time +[SCHEDULER ITER22 LR0.07776]: 720043 words processed Tue Nov 17 02:15:49 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.920734. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20683 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96721 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75829 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42137 clock time +[SCHEDULER ITER22 LR0.07776]: 760012 words processed Tue Nov 17 02:16:04 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.918822. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23640 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97318 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77081 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42451 clock time +[SCHEDULER ITER22 LR0.07776]: 800113 words processed Tue Nov 17 02:16:19 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.918464. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26754 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98893 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77631 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43179 clock time +[SCHEDULER ITER22 LR0.07776]: 840089 words processed Tue Nov 17 02:16:34 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.917255. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20786 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96696 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76022 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42168 clock time +[SCHEDULER ITER22 LR0.07776]: 880052 words processed Tue Nov 17 02:16:49 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-1.916834. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24615 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97516 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77585 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42352 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER22 LR0.07776]: Displaying result: -[SCHEDULER ITER22 LR0.07776]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 88.094386760042> <PPL_OOV 80.0732601848> <LOGP -1769461.0520283> +[SCHEDULER ITER22 LR0.07776]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 90.91666059119> <PPL_OOV 82.567145086936> <LOGP -1781842.9517444> [SCHEDULER ITER22 LR0.07776]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER22 LR0.07776]: shuffling training file ===PEEK ON TEST 22=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER22 LR0.07776]: 40087 words processed Mon Nov 16 21:24:09 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-2.115979. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.84544 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87784 clock time +[SCHEDULER ITER22 LR0.07776]: 40087 words processed Tue Nov 17 02:17:00 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-2.109316. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72884 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79022 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER22 LR0.07776]: Displaying result: -[SCHEDULER ITER22 LR0.07776]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 150.34388392672> <PPL_OOV 131.1224071462> <LOGP -174560.10797298> +[SCHEDULER ITER22 LR0.07776]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 148.47280507404> <PPL_OOV 129.37769514195> <LOGP -174080.57116102> [SCHEDULER ITER22 LR0.07776]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 22=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER22 LR0.07776]: 40095 words processed Mon Nov 16 21:24:22 2015. - [SCHEDULER ITER22 LR0.07776]: log prob per sample :-2.188293. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69986 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78278 clock time +[SCHEDULER ITER22 LR0.07776]: 40095 words processed Tue Nov 17 02:17:13 2015. + [SCHEDULER ITER22 LR0.07776]: log prob per sample :-2.183313. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70671 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77644 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER22 LR0.07776]: Displaying result: -[SCHEDULER ITER22 LR0.07776]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 153.99594513805> <PPL_OOV 137.63667528429> <LOGP -157753.03262785> +[SCHEDULER ITER22 LR0.07776]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 153.3005736827> <PPL_OOV 136.93758076193> <LOGP -157589.91074068> [SCHEDULER ITER22 LR0.07776]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER22 LR0.07776]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.22... @@ -4318,27 +4810,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1890622 +reading chunk 1 from 1893435 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1895476 +reading chunk 2 from 1898284 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48184985 +reading chunk 3 from 48154857 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48301204 +reading chunk 4 from 48271120 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94328908 +reading chunk 5 from 94306015 [SCHEDULER ITER23 LR0.046656]: preparing parameters end. [SCHEDULER ITER23 LR0.046656]: preparing layers... -(21:24:33 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:24:33 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:24:33 2015-11-16)[nerv] info: create layer: combinerL1 -(21:24:33 2015-11-16)[nerv] info: create layer: outputL -(21:24:33 2015-11-16)[nerv] info: create layer: softmaxL -(21:24:33 2015-11-16)[nerv] info: create layer: selectL1 +(02:17:25 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:17:25 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:17:25 2015-11-17)[nerv] info: create layer: combinerL1 +(02:17:25 2015-11-17)[nerv] info: create layer: outputL +(02:17:25 2015-11-17)[nerv] info: create layer: softmaxL +(02:17:25 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER23 LR0.046656]: preparing layers end. [SCHEDULER ITER23 LR0.046656]: Generate and initing TNN ... <input> selectL1 0 @@ -4366,164 +4858,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 23 LR 0.046656=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER23 LR0.046656]: 40092 words processed Mon Nov 16 21:24:48 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.933322. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.43060 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02538 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85628 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45642 clock time -[SCHEDULER ITER23 LR0.046656]: 80099 words processed Mon Nov 16 21:25:03 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.929559. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22654 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97019 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78185 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42971 clock time -[SCHEDULER ITER23 LR0.046656]: 120004 words processed Mon Nov 16 21:25:18 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.923031. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32611 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99042 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82348 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43949 clock time -[SCHEDULER ITER23 LR0.046656]: 160114 words processed Mon Nov 16 21:25:33 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.924075. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27485 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98920 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79461 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43725 clock time -[SCHEDULER ITER23 LR0.046656]: 200066 words processed Mon Nov 16 21:25:48 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.923074. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17493 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96409 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75168 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42667 clock time -[SCHEDULER ITER23 LR0.046656]: 240045 words processed Mon Nov 16 21:26:03 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.919194. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20807 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96601 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42754 clock time -[SCHEDULER ITER23 LR0.046656]: 280057 words processed Mon Nov 16 21:26:18 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.914950. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38218 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01743 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82897 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45330 clock time -[SCHEDULER ITER23 LR0.046656]: 320106 words processed Mon Nov 16 21:26:33 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.912753. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34983 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01235 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81060 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45143 clock time -[SCHEDULER ITER23 LR0.046656]: 360024 words processed Mon Nov 16 21:26:48 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.910919. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31415 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99817 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80388 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44630 clock time -[SCHEDULER ITER23 LR0.046656]: 400089 words processed Mon Nov 16 21:27:03 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.907852. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38152 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01851 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82635 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45193 clock time -[SCHEDULER ITER23 LR0.046656]: 440067 words processed Mon Nov 16 21:27:18 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.908329. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24014 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97961 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78054 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43533 clock time -[SCHEDULER ITER23 LR0.046656]: 480051 words processed Mon Nov 16 21:27:32 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.908451. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17048 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96364 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75443 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42763 clock time -[SCHEDULER ITER23 LR0.046656]: 520140 words processed Mon Nov 16 21:27:47 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.907763. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32453 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00377 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81899 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44426 clock time -[SCHEDULER ITER23 LR0.046656]: 560132 words processed Mon Nov 16 21:28:02 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.906947. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19512 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96572 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77019 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42856 clock time -[SCHEDULER ITER23 LR0.046656]: 600118 words processed Mon Nov 16 21:28:17 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.904446. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19011 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96633 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76496 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42736 clock time -[SCHEDULER ITER23 LR0.046656]: 640090 words processed Mon Nov 16 21:28:32 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.902126. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29103 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99197 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79511 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44366 clock time -[SCHEDULER ITER23 LR0.046656]: 680075 words processed Mon Nov 16 21:28:47 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.900667. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27285 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98424 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80619 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43415 clock time -[SCHEDULER ITER23 LR0.046656]: 720043 words processed Mon Nov 16 21:29:02 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.899347. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27555 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98505 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79897 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43413 clock time -[SCHEDULER ITER23 LR0.046656]: 760012 words processed Mon Nov 16 21:29:16 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.897339. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16690 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96549 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74945 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42699 clock time -[SCHEDULER ITER23 LR0.046656]: 800113 words processed Mon Nov 16 21:29:31 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.896996. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21779 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97776 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76997 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43602 clock time -[SCHEDULER ITER23 LR0.046656]: 840089 words processed Mon Nov 16 21:29:45 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.895727. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16285 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96054 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75017 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42617 clock time -[SCHEDULER ITER23 LR0.046656]: 880052 words processed Mon Nov 16 21:30:00 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.895135. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16826 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96469 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74942 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42590 clock time +[SCHEDULER ITER23 LR0.046656]: 40092 words processed Tue Nov 17 02:17:39 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.943328. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26098 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97876 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77861 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43042 clock time +[SCHEDULER ITER23 LR0.046656]: 80099 words processed Tue Nov 17 02:17:54 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.939708. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19670 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95266 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76176 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41839 clock time +[SCHEDULER ITER23 LR0.046656]: 120004 words processed Tue Nov 17 02:18:09 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.934698. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24449 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96494 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78132 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42240 clock time +[SCHEDULER ITER23 LR0.046656]: 160114 words processed Tue Nov 17 02:18:24 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.935900. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26015 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97477 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78166 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43001 clock time +[SCHEDULER ITER23 LR0.046656]: 200066 words processed Tue Nov 17 02:18:39 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.935174. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26162 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97042 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78443 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42786 clock time +[SCHEDULER ITER23 LR0.046656]: 240045 words processed Tue Nov 17 02:18:54 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.930901. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19776 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95589 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76400 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41995 clock time +[SCHEDULER ITER23 LR0.046656]: 280057 words processed Tue Nov 17 02:19:09 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.926836. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25284 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97577 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77774 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42987 clock time +[SCHEDULER ITER23 LR0.046656]: 320106 words processed Tue Nov 17 02:19:24 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.924857. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27265 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78572 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43285 clock time +[SCHEDULER ITER23 LR0.046656]: 360024 words processed Tue Nov 17 02:19:39 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.923119. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22791 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96260 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77511 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42402 clock time +[SCHEDULER ITER23 LR0.046656]: 400089 words processed Tue Nov 17 02:19:54 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.920062. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23397 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97119 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77566 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42765 clock time +[SCHEDULER ITER23 LR0.046656]: 440067 words processed Tue Nov 17 02:20:09 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.920597. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19054 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95310 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76402 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41949 clock time +[SCHEDULER ITER23 LR0.046656]: 480051 words processed Tue Nov 17 02:20:24 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.921059. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21391 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96242 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77062 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42185 clock time +[SCHEDULER ITER23 LR0.046656]: 520140 words processed Tue Nov 17 02:20:39 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.920521. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00905 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80941 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44577 clock time +[SCHEDULER ITER23 LR0.046656]: 560132 words processed Tue Nov 17 02:20:54 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.920070. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26295 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97742 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78371 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43213 clock time +[SCHEDULER ITER23 LR0.046656]: 600118 words processed Tue Nov 17 02:21:09 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.917606. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23433 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96548 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77851 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42545 clock time +[SCHEDULER ITER23 LR0.046656]: 640090 words processed Tue Nov 17 02:21:24 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.915317. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24519 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96710 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78193 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42541 clock time +[SCHEDULER ITER23 LR0.046656]: 680075 words processed Tue Nov 17 02:21:39 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.913860. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32276 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80868 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43676 clock time +[SCHEDULER ITER23 LR0.046656]: 720043 words processed Tue Nov 17 02:21:54 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.912697. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28838 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97653 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80062 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43125 clock time +[SCHEDULER ITER23 LR0.046656]: 760012 words processed Tue Nov 17 02:22:09 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.910771. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19920 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95701 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76615 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42009 clock time +[SCHEDULER ITER23 LR0.046656]: 800113 words processed Tue Nov 17 02:22:24 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.910385. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24158 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97376 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77657 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42811 clock time +[SCHEDULER ITER23 LR0.046656]: 840089 words processed Tue Nov 17 02:22:39 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.909166. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27981 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98300 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43241 clock time +[SCHEDULER ITER23 LR0.046656]: 880052 words processed Tue Nov 17 02:22:54 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-1.908725. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27804 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97789 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79635 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42772 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER23 LR0.046656]: Displaying result: -[SCHEDULER ITER23 LR0.046656]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.370829580211> <PPL_OOV 78.540181305621> <LOGP -1761656.5837458> +[SCHEDULER ITER23 LR0.046656]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 89.193926668816> <PPL_OOV 81.037835193187> <LOGP -1774295.2121013> [SCHEDULER ITER23 LR0.046656]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER23 LR0.046656]: shuffling training file ===PEEK ON TEST 23=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER23 LR0.046656]: 40087 words processed Mon Nov 16 21:30:11 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-2.112277. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77059 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81826 clock time +[SCHEDULER ITER23 LR0.046656]: 40087 words processed Tue Nov 17 02:23:05 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-2.105861. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70428 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77841 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER23 LR0.046656]: Displaying result: -[SCHEDULER ITER23 LR0.046656]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 149.10919738157> <PPL_OOV 130.08618277319> <LOGP -174276.07534413> +[SCHEDULER ITER23 LR0.046656]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 147.17493465036> <PPL_OOV 128.4030207272> <LOGP -173809.85709805> [SCHEDULER ITER23 LR0.046656]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 23=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER23 LR0.046656]: 40095 words processed Mon Nov 16 21:30:24 2015. - [SCHEDULER ITER23 LR0.046656]: log prob per sample :-2.184797. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.76768 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82157 clock time +[SCHEDULER ITER23 LR0.046656]: 40095 words processed Tue Nov 17 02:23:18 2015. + [SCHEDULER ITER23 LR0.046656]: log prob per sample :-2.180019. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69577 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77204 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER23 LR0.046656]: Displaying result: -[SCHEDULER ITER23 LR0.046656]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 152.58447301765> <PPL_OOV 136.42030747027> <LOGP -157468.67681237> +[SCHEDULER ITER23 LR0.046656]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 151.84224283761> <PPL_OOV 135.76813395708> <LOGP -157315.16933881> [SCHEDULER ITER23 LR0.046656]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER23 LR0.046656]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.23... @@ -4532,27 +5024,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46027630 +reading chunk 1 from 46034688 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46032483 +reading chunk 2 from 46039537 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92321690 +reading chunk 3 from 92295942 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92437904 +reading chunk 4 from 92412200 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94328313 +reading chunk 5 from 94305436 [SCHEDULER ITER24 LR0.0279936]: preparing parameters end. [SCHEDULER ITER24 LR0.0279936]: preparing layers... -(21:30:36 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:30:36 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:30:36 2015-11-16)[nerv] info: create layer: combinerL1 -(21:30:36 2015-11-16)[nerv] info: create layer: outputL -(21:30:36 2015-11-16)[nerv] info: create layer: softmaxL -(21:30:36 2015-11-16)[nerv] info: create layer: selectL1 +(02:23:29 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:23:29 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:23:29 2015-11-17)[nerv] info: create layer: combinerL1 +(02:23:29 2015-11-17)[nerv] info: create layer: outputL +(02:23:29 2015-11-17)[nerv] info: create layer: softmaxL +(02:23:29 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER24 LR0.0279936]: preparing layers end. [SCHEDULER ITER24 LR0.0279936]: Generate and initing TNN ... <input> selectL1 0 @@ -4580,164 +5072,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 24 LR 0.027994=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER24 LR0.0279936]: 40092 words processed Mon Nov 16 21:30:50 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.927572. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29372 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96722 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81640 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44418 clock time -[SCHEDULER ITER24 LR0.0279936]: 80099 words processed Mon Nov 16 21:31:05 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.923793. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27562 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95174 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82158 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43672 clock time -[SCHEDULER ITER24 LR0.0279936]: 120004 words processed Mon Nov 16 21:31:20 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.917436. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25729 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94434 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82537 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43113 clock time -[SCHEDULER ITER24 LR0.0279936]: 160114 words processed Mon Nov 16 21:31:35 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.918508. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29215 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96581 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82217 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44249 clock time -[SCHEDULER ITER24 LR0.0279936]: 200066 words processed Mon Nov 16 21:31:49 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.917544. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16679 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93220 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77049 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42816 clock time -[SCHEDULER ITER24 LR0.0279936]: 240045 words processed Mon Nov 16 21:32:04 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.913702. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17291 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93194 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77446 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42967 clock time -[SCHEDULER ITER24 LR0.0279936]: 280057 words processed Mon Nov 16 21:32:19 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.909467. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29023 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96128 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82053 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44194 clock time -[SCHEDULER ITER24 LR0.0279936]: 320106 words processed Mon Nov 16 21:32:34 2015. +[SCHEDULER ITER24 LR0.0279936]: 40092 words processed Tue Nov 17 02:23:43 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.937618. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31006 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00202 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78357 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43654 clock time +[SCHEDULER ITER24 LR0.0279936]: 80099 words processed Tue Nov 17 02:23:58 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.934095. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24558 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97686 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76685 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42357 clock time +[SCHEDULER ITER24 LR0.0279936]: 120004 words processed Tue Nov 17 02:24:13 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.929226. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25484 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97572 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77452 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42280 clock time +[SCHEDULER ITER24 LR0.0279936]: 160114 words processed Tue Nov 17 02:24:28 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.930426. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00836 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80343 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43965 clock time +[SCHEDULER ITER24 LR0.0279936]: 200066 words processed Tue Nov 17 02:24:43 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.929773. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23130 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97690 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75995 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42254 clock time +[SCHEDULER ITER24 LR0.0279936]: 240045 words processed Tue Nov 17 02:24:58 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.925530. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29263 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98990 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78602 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42925 clock time +[SCHEDULER ITER24 LR0.0279936]: 280057 words processed Tue Nov 17 02:25:13 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.921505. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29217 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99392 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78429 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43080 clock time +[SCHEDULER ITER24 LR0.0279936]: 320106 words processed Tue Nov 17 02:25:28 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.919564. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36306 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.01089 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80593 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44205 clock time +[SCHEDULER ITER24 LR0.0279936]: 360024 words processed Tue Nov 17 02:25:43 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.917825. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00346 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80996 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43831 clock time +[SCHEDULER ITER24 LR0.0279936]: 400089 words processed Tue Nov 17 02:25:58 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.914773. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33067 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00315 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79572 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43796 clock time +[SCHEDULER ITER24 LR0.0279936]: 440067 words processed Tue Nov 17 02:26:13 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.915313. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35650 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81070 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44042 clock time +[SCHEDULER ITER24 LR0.0279936]: 480051 words processed Tue Nov 17 02:26:28 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.915779. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32763 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99708 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80040 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43528 clock time +[SCHEDULER ITER24 LR0.0279936]: 520140 words processed Tue Nov 17 02:26:43 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.915223. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25704 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98821 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76955 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42952 clock time +[SCHEDULER ITER24 LR0.0279936]: 560132 words processed Tue Nov 17 02:26:58 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.914765. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23893 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97922 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76821 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42359 clock time +[SCHEDULER ITER24 LR0.0279936]: 600118 words processed Tue Nov 17 02:27:13 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.912281. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22913 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97521 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76487 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42246 clock time +[SCHEDULER ITER24 LR0.0279936]: 640090 words processed Tue Nov 17 02:27:28 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.909977. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31716 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99499 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79637 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43459 clock time +[SCHEDULER ITER24 LR0.0279936]: 680075 words processed Tue Nov 17 02:27:43 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.908495. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26854 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98405 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77917 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42941 clock time +[SCHEDULER ITER24 LR0.0279936]: 720043 words processed Tue Nov 17 02:27:58 2015. [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.907301. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22057 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95197 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78918 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43668 clock time -[SCHEDULER ITER24 LR0.0279936]: 360024 words processed Mon Nov 16 21:32:49 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.905454. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36237 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96948 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86266 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44618 clock time -[SCHEDULER ITER24 LR0.0279936]: 400089 words processed Mon Nov 16 21:33:04 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.902393. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39099 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98331 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87418 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45318 clock time -[SCHEDULER ITER24 LR0.0279936]: 440067 words processed Mon Nov 16 21:33:19 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.902894. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27378 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95025 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83149 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43587 clock time -[SCHEDULER ITER24 LR0.0279936]: 480051 words processed Mon Nov 16 21:33:33 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.902990. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15579 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77021 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42697 clock time -[SCHEDULER ITER24 LR0.0279936]: 520140 words processed Mon Nov 16 21:33:48 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.902279. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21845 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94832 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79380 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43759 clock time -[SCHEDULER ITER24 LR0.0279936]: 560132 words processed Mon Nov 16 21:34:03 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.901444. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18088 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93598 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78271 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42838 clock time -[SCHEDULER ITER24 LR0.0279936]: 600118 words processed Mon Nov 16 21:34:18 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.898944. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28395 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95327 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83804 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43478 clock time -[SCHEDULER ITER24 LR0.0279936]: 640090 words processed Mon Nov 16 21:34:33 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.896618. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95035 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82729 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43462 clock time -[SCHEDULER ITER24 LR0.0279936]: 680075 words processed Mon Nov 16 21:34:48 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.895142. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18574 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93844 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77808 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43477 clock time -[SCHEDULER ITER24 LR0.0279936]: 720043 words processed Mon Nov 16 21:35:03 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.893792. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27391 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95735 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81735 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44041 clock time -[SCHEDULER ITER24 LR0.0279936]: 760012 words processed Mon Nov 16 21:35:18 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.891750. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20482 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93922 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79458 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43129 clock time -[SCHEDULER ITER24 LR0.0279936]: 800113 words processed Mon Nov 16 21:35:33 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.891385. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24458 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95290 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81107 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43856 clock time -[SCHEDULER ITER24 LR0.0279936]: 840089 words processed Mon Nov 16 21:35:48 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.890073. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20478 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93956 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79600 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42966 clock time -[SCHEDULER ITER24 LR0.0279936]: 880052 words processed Mon Nov 16 21:36:02 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.889435. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16192 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93020 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77294 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42815 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31719 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99493 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79369 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43407 clock time +[SCHEDULER ITER24 LR0.0279936]: 760012 words processed Tue Nov 17 02:28:13 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.905351. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23426 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97649 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76718 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42283 clock time +[SCHEDULER ITER24 LR0.0279936]: 800113 words processed Tue Nov 17 02:28:28 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.904930. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28478 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99595 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77862 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43295 clock time +[SCHEDULER ITER24 LR0.0279936]: 840089 words processed Tue Nov 17 02:28:43 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.903683. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20679 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96903 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75713 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42071 clock time +[SCHEDULER ITER24 LR0.0279936]: 880052 words processed Tue Nov 17 02:28:58 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-1.903209. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20872 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96944 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75613 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42097 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER24 LR0.0279936]: Displaying result: -[SCHEDULER ITER24 LR0.0279936]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 85.22026591435> <PPL_OOV 77.514508595711> <LOGP -1756349.6518786> +[SCHEDULER ITER24 LR0.0279936]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 88.042301854058> <PPL_OOV 80.012919091616> <LOGP -1769156.7081029> [SCHEDULER ITER24 LR0.0279936]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER24 LR0.0279936]: shuffling training file ===PEEK ON TEST 24=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER24 LR0.0279936]: 40087 words processed Mon Nov 16 21:36:14 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-2.109803. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.76426 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83310 clock time +[SCHEDULER ITER24 LR0.0279936]: 40087 words processed Tue Nov 17 02:29:09 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-2.103746. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73719 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79546 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER24 LR0.0279936]: Displaying result: -[SCHEDULER ITER24 LR0.0279936]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 148.18558701419> <PPL_OOV 129.38121442773> <LOGP -174081.54493649> +[SCHEDULER ITER24 LR0.0279936]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.22167024225> <PPL_OOV 127.78132116482> <LOGP -173636.10564745> [SCHEDULER ITER24 LR0.0279936]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 24=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER24 LR0.0279936]: 40095 words processed Mon Nov 16 21:36:28 2015. - [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-2.182444. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.76095 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83160 clock time +[SCHEDULER ITER24 LR0.0279936]: 40095 words processed Tue Nov 17 02:29:22 2015. + [SCHEDULER ITER24 LR0.0279936]: log prob per sample :-2.177995. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.78790 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82669 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER24 LR0.0279936]: Displaying result: -[SCHEDULER ITER24 LR0.0279936]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 151.56327400662> <PPL_OOV 135.60786959211> <LOGP -157277.33372447> +[SCHEDULER ITER24 LR0.0279936]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 150.79123361395> <PPL_OOV 135.00968447248> <LOGP -157135.71666799> [SCHEDULER ITER24 LR0.0279936]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER24 LR0.0279936]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.24... @@ -4746,27 +5238,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1890306 +reading chunk 1 from 1893126 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1895158 +reading chunk 2 from 1897975 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48184112 +reading chunk 3 from 48154211 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48300326 +reading chunk 4 from 48270468 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94327911 +reading chunk 5 from 94304939 [SCHEDULER ITER25 LR0.01679616]: preparing parameters end. [SCHEDULER ITER25 LR0.01679616]: preparing layers... -(21:36:40 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:36:40 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:36:40 2015-11-16)[nerv] info: create layer: combinerL1 -(21:36:40 2015-11-16)[nerv] info: create layer: outputL -(21:36:40 2015-11-16)[nerv] info: create layer: softmaxL -(21:36:40 2015-11-16)[nerv] info: create layer: selectL1 +(02:29:33 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:29:33 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:29:33 2015-11-17)[nerv] info: create layer: combinerL1 +(02:29:33 2015-11-17)[nerv] info: create layer: outputL +(02:29:33 2015-11-17)[nerv] info: create layer: softmaxL +(02:29:33 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER25 LR0.01679616]: preparing layers end. [SCHEDULER ITER25 LR0.01679616]: Generate and initing TNN ... <input> selectL1 0 @@ -4794,164 +5286,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 25 LR 0.016796=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER25 LR0.01679616]: 40092 words processed Mon Nov 16 21:36:55 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.923208. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27808 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99055 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43951 clock time -[SCHEDULER ITER25 LR0.01679616]: 80099 words processed Mon Nov 16 21:37:10 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.919417. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23524 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97744 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77503 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43228 clock time -[SCHEDULER ITER25 LR0.01679616]: 120004 words processed Mon Nov 16 21:37:25 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.913209. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19035 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96580 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75638 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42841 clock time -[SCHEDULER ITER25 LR0.01679616]: 160114 words processed Mon Nov 16 21:37:40 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.914297. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25164 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98748 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77835 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43606 clock time -[SCHEDULER ITER25 LR0.01679616]: 200066 words processed Mon Nov 16 21:37:55 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.913380. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36925 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00198 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84568 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44147 clock time -[SCHEDULER ITER25 LR0.01679616]: 240045 words processed Mon Nov 16 21:38:10 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.909582. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32865 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99385 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82956 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43906 clock time -[SCHEDULER ITER25 LR0.01679616]: 280057 words processed Mon Nov 16 21:38:25 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.905394. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30278 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99811 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80327 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44108 clock time -[SCHEDULER ITER25 LR0.01679616]: 320106 words processed Mon Nov 16 21:38:40 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.903267. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31148 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99837 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81722 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43881 clock time -[SCHEDULER ITER25 LR0.01679616]: 360024 words processed Mon Nov 16 21:38:55 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.901437. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18041 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96556 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75637 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42611 clock time -[SCHEDULER ITER25 LR0.01679616]: 400089 words processed Mon Nov 16 21:39:10 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.898384. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22891 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98562 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76707 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43703 clock time -[SCHEDULER ITER25 LR0.01679616]: 440067 words processed Mon Nov 16 21:39:25 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.898922. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17740 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96629 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75729 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42587 clock time -[SCHEDULER ITER25 LR0.01679616]: 480051 words processed Mon Nov 16 21:39:40 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.899016. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33944 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99705 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83188 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44089 clock time -[SCHEDULER ITER25 LR0.01679616]: 520140 words processed Mon Nov 16 21:39:55 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.898314. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35601 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83057 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44649 clock time -[SCHEDULER ITER25 LR0.01679616]: 560132 words processed Mon Nov 16 21:40:10 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.897483. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31315 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98899 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82788 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43626 clock time -[SCHEDULER ITER25 LR0.01679616]: 600118 words processed Mon Nov 16 21:40:25 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.895000. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97175 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77197 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42846 clock time -[SCHEDULER ITER25 LR0.01679616]: 640090 words processed Mon Nov 16 21:40:40 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.892683. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99863 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84076 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43990 clock time -[SCHEDULER ITER25 LR0.01679616]: 680075 words processed Mon Nov 16 21:40:55 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.891213. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28306 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98603 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81047 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43297 clock time -[SCHEDULER ITER25 LR0.01679616]: 720043 words processed Mon Nov 16 21:41:10 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.889857. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24383 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97628 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79443 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42965 clock time -[SCHEDULER ITER25 LR0.01679616]: 760012 words processed Mon Nov 16 21:41:25 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.887813. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23900 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97564 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79127 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42940 clock time -[SCHEDULER ITER25 LR0.01679616]: 800113 words processed Mon Nov 16 21:41:40 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.887451. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30064 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00949 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78820 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45058 clock time -[SCHEDULER ITER25 LR0.01679616]: 840089 words processed Mon Nov 16 21:41:55 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.886122. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29066 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00731 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78502 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44161 clock time -[SCHEDULER ITER25 LR0.01679616]: 880052 words processed Mon Nov 16 21:42:10 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.885470. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27025 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99326 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79034 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43514 clock time +[SCHEDULER ITER25 LR0.01679616]: 40092 words processed Tue Nov 17 02:29:47 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.933340. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32239 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79678 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43848 clock time +[SCHEDULER ITER25 LR0.01679616]: 80099 words processed Tue Nov 17 02:30:02 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.929899. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20877 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95715 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76790 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41902 clock time +[SCHEDULER ITER25 LR0.01679616]: 120004 words processed Tue Nov 17 02:30:17 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.925183. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20150 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95826 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75905 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41839 clock time +[SCHEDULER ITER25 LR0.01679616]: 160114 words processed Tue Nov 17 02:30:32 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.926365. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24361 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97460 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77340 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42676 clock time +[SCHEDULER ITER25 LR0.01679616]: 200066 words processed Tue Nov 17 02:30:47 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.925781. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19704 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95694 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76191 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41855 clock time +[SCHEDULER ITER25 LR0.01679616]: 240045 words processed Tue Nov 17 02:31:02 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.921576. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21668 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96442 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76698 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42182 clock time +[SCHEDULER ITER25 LR0.01679616]: 280057 words processed Tue Nov 17 02:31:17 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.917590. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28327 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98338 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79064 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43197 clock time +[SCHEDULER ITER25 LR0.01679616]: 320106 words processed Tue Nov 17 02:31:32 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.915687. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34137 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99859 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80954 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43801 clock time +[SCHEDULER ITER25 LR0.01679616]: 360024 words processed Tue Nov 17 02:31:47 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.913957. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20835 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96245 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76632 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41984 clock time +[SCHEDULER ITER25 LR0.01679616]: 400089 words processed Tue Nov 17 02:32:02 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.910927. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25525 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97743 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78184 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42918 clock time +[SCHEDULER ITER25 LR0.01679616]: 440067 words processed Tue Nov 17 02:32:17 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.911484. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20215 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96039 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76615 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42016 clock time +[SCHEDULER ITER25 LR0.01679616]: 480051 words processed Tue Nov 17 02:32:32 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.911963. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23099 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96703 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77768 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42295 clock time +[SCHEDULER ITER25 LR0.01679616]: 520140 words processed Tue Nov 17 02:32:47 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.911406. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25521 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42914 clock time +[SCHEDULER ITER25 LR0.01679616]: 560132 words processed Tue Nov 17 02:33:02 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.910966. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31953 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98589 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80748 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43417 clock time +[SCHEDULER ITER25 LR0.01679616]: 600118 words processed Tue Nov 17 02:33:17 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.908482. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19172 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95805 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76083 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41851 clock time +[SCHEDULER ITER25 LR0.01679616]: 640090 words processed Tue Nov 17 02:33:32 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.906177. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19725 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95894 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76348 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41998 clock time +[SCHEDULER ITER25 LR0.01679616]: 680075 words processed Tue Nov 17 02:33:47 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.904691. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95623 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76322 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41815 clock time +[SCHEDULER ITER25 LR0.01679616]: 720043 words processed Tue Nov 17 02:34:02 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.903487. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19706 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96004 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76099 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41973 clock time +[SCHEDULER ITER25 LR0.01679616]: 760012 words processed Tue Nov 17 02:34:17 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.901534. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19780 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95810 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76590 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41880 clock time +[SCHEDULER ITER25 LR0.01679616]: 800113 words processed Tue Nov 17 02:34:32 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.901109. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26472 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98323 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78151 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43077 clock time +[SCHEDULER ITER25 LR0.01679616]: 840089 words processed Tue Nov 17 02:34:47 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.899860. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19617 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95840 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76348 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41920 clock time +[SCHEDULER ITER25 LR0.01679616]: 880052 words processed Tue Nov 17 02:35:02 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-1.899378. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96017 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76606 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42131 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER25 LR0.01679616]: Displaying result: -[SCHEDULER ITER25 LR0.01679616]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 84.428540798804> <PPL_OOV 76.809572863182> <LOGP -1752661.3698032> +[SCHEDULER ITER25 LR0.01679616]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 87.251020327347> <PPL_OOV 79.309227094631> <LOGP -1765590.4352236> [SCHEDULER ITER25 LR0.01679616]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER25 LR0.01679616]: shuffling training file ===PEEK ON TEST 25=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER25 LR0.01679616]: 40087 words processed Mon Nov 16 21:42:22 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-2.108055. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.84361 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87344 clock time +[SCHEDULER ITER25 LR0.01679616]: 40087 words processed Tue Nov 17 02:35:13 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-2.102266. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70299 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77708 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER25 LR0.01679616]: Displaying result: -[SCHEDULER ITER25 LR0.01679616]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 147.48955179622> <PPL_OOV 128.86335181656> <LOGP -173937.96837325> +[SCHEDULER ITER25 LR0.01679616]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 145.57836171145> <PPL_OOV 127.34096923105> <LOGP -173512.52469422> [SCHEDULER ITER25 LR0.01679616]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 25=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER25 LR0.01679616]: 40095 words processed Mon Nov 16 21:42:36 2015. - [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-2.180677. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73914 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80862 clock time +[SCHEDULER ITER25 LR0.01679616]: 40095 words processed Tue Nov 17 02:35:26 2015. + [SCHEDULER ITER25 LR0.01679616]: log prob per sample :-2.176467. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72177 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79015 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER25 LR0.01679616]: Displaying result: -[SCHEDULER ITER25 LR0.01679616]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 150.77220031057> <PPL_OOV 134.99308858218> <LOGP -157131.77874203> +[SCHEDULER ITER25 LR0.01679616]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 150.0332906762> <PPL_OOV 134.43926251013> <LOGP -157000.08671493> [SCHEDULER ITER25 LR0.01679616]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER25 LR0.01679616]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.25... @@ -4960,27 +5452,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46027486 +reading chunk 1 from 46034350 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46032336 +reading chunk 2 from 46039198 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92321123 +reading chunk 3 from 92295302 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92437338 +reading chunk 4 from 92411556 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94327574 +reading chunk 5 from 94304612 [SCHEDULER ITER26 LR0.010077696]: preparing parameters end. [SCHEDULER ITER26 LR0.010077696]: preparing layers... -(21:42:47 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:42:47 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:42:47 2015-11-16)[nerv] info: create layer: combinerL1 -(21:42:47 2015-11-16)[nerv] info: create layer: outputL -(21:42:47 2015-11-16)[nerv] info: create layer: softmaxL -(21:42:47 2015-11-16)[nerv] info: create layer: selectL1 +(02:35:37 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:35:37 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:35:37 2015-11-17)[nerv] info: create layer: combinerL1 +(02:35:37 2015-11-17)[nerv] info: create layer: outputL +(02:35:37 2015-11-17)[nerv] info: create layer: softmaxL +(02:35:37 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER26 LR0.010077696]: preparing layers end. [SCHEDULER ITER26 LR0.010077696]: Generate and initing TNN ... <input> selectL1 0 @@ -5008,164 +5500,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 26 LR 0.010078=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER26 LR0.010077696]: 40092 words processed Mon Nov 16 21:43:02 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.920393. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28153 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96072 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81616 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44630 clock time -[SCHEDULER ITER26 LR0.010077696]: 80099 words processed Mon Nov 16 21:43:17 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.916572. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19306 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93521 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78438 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42983 clock time -[SCHEDULER ITER26 LR0.010077696]: 120004 words processed Mon Nov 16 21:43:32 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.910485. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30470 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95864 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82799 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44355 clock time -[SCHEDULER ITER26 LR0.010077696]: 160114 words processed Mon Nov 16 21:43:47 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.911575. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96489 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83005 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44568 clock time -[SCHEDULER ITER26 LR0.010077696]: 200066 words processed Mon Nov 16 21:44:02 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.910704. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25513 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94238 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82516 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43332 clock time -[SCHEDULER ITER26 LR0.010077696]: 240045 words processed Mon Nov 16 21:44:17 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.906925. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18064 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93152 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43142 clock time -[SCHEDULER ITER26 LR0.010077696]: 280057 words processed Mon Nov 16 21:44:32 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.902777. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22353 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94530 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79585 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43698 clock time -[SCHEDULER ITER26 LR0.010077696]: 320106 words processed Mon Nov 16 21:44:47 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.900686. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20239 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94432 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78143 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43656 clock time -[SCHEDULER ITER26 LR0.010077696]: 360024 words processed Mon Nov 16 21:45:02 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.898858. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18606 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93249 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78490 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43095 clock time -[SCHEDULER ITER26 LR0.010077696]: 400089 words processed Mon Nov 16 21:45:17 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.895803. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21919 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94961 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78914 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43940 clock time -[SCHEDULER ITER26 LR0.010077696]: 440067 words processed Mon Nov 16 21:45:31 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.896372. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16197 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93134 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77086 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42857 clock time -[SCHEDULER ITER26 LR0.010077696]: 480051 words processed Mon Nov 16 21:45:45 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.896467. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16589 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93048 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77382 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42891 clock time -[SCHEDULER ITER26 LR0.010077696]: 520140 words processed Mon Nov 16 21:46:00 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.895771. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22682 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95144 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79498 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43805 clock time -[SCHEDULER ITER26 LR0.010077696]: 560132 words processed Mon Nov 16 21:46:15 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.894951. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20550 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93912 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78845 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43555 clock time -[SCHEDULER ITER26 LR0.010077696]: 600118 words processed Mon Nov 16 21:46:30 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.892485. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31142 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95919 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84282 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44207 clock time -[SCHEDULER ITER26 LR0.010077696]: 640090 words processed Mon Nov 16 21:46:45 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.890172. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94833 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80967 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43841 clock time -[SCHEDULER ITER26 LR0.010077696]: 680075 words processed Mon Nov 16 21:47:00 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.888707. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23265 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94233 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79921 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43991 clock time -[SCHEDULER ITER26 LR0.010077696]: 720043 words processed Mon Nov 16 21:47:15 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.887352. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17209 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93380 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76914 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43137 clock time -[SCHEDULER ITER26 LR0.010077696]: 760012 words processed Mon Nov 16 21:47:30 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.885304. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16540 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92808 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77617 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42964 clock time -[SCHEDULER ITER26 LR0.010077696]: 800113 words processed Mon Nov 16 21:47:45 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.884942. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27983 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96469 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81050 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44758 clock time -[SCHEDULER ITER26 LR0.010077696]: 840089 words processed Mon Nov 16 21:48:00 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.883602. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24576 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94721 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80543 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44016 clock time -[SCHEDULER ITER26 LR0.010077696]: 880052 words processed Mon Nov 16 21:48:15 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.882941. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21524 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94225 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78659 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43695 clock time +[SCHEDULER ITER26 LR0.010077696]: 40092 words processed Tue Nov 17 02:35:52 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.930522. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38963 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.01549 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81137 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44577 clock time +[SCHEDULER ITER26 LR0.010077696]: 80099 words processed Tue Nov 17 02:36:07 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.927158. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27655 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98305 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77545 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42724 clock time +[SCHEDULER ITER26 LR0.010077696]: 120004 words processed Tue Nov 17 02:36:22 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.922568. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21774 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96666 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75408 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42107 clock time +[SCHEDULER ITER26 LR0.010077696]: 160114 words processed Tue Nov 17 02:36:37 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.923726. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26053 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76829 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42976 clock time +[SCHEDULER ITER26 LR0.010077696]: 200066 words processed Tue Nov 17 02:36:52 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.923204. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20620 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96542 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75190 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42057 clock time +[SCHEDULER ITER26 LR0.010077696]: 240045 words processed Tue Nov 17 02:37:07 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.919023. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21935 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96726 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75824 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42225 clock time +[SCHEDULER ITER26 LR0.010077696]: 280057 words processed Tue Nov 17 02:37:22 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.915064. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.41502 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.01846 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82207 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44815 clock time +[SCHEDULER ITER26 LR0.010077696]: 320106 words processed Tue Nov 17 02:37:37 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.913183. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.40402 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.01815 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82066 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44669 clock time +[SCHEDULER ITER26 LR0.010077696]: 360024 words processed Tue Nov 17 02:37:52 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.911456. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27822 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98173 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77980 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42867 clock time +[SCHEDULER ITER26 LR0.010077696]: 400089 words processed Tue Nov 17 02:38:07 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.908451. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36520 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00814 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44232 clock time +[SCHEDULER ITER26 LR0.010077696]: 440067 words processed Tue Nov 17 02:38:22 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.909026. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30775 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99090 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79151 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43091 clock time +[SCHEDULER ITER26 LR0.010077696]: 480051 words processed Tue Nov 17 02:38:37 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.909514. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19978 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96594 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75130 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42113 clock time +[SCHEDULER ITER26 LR0.010077696]: 520140 words processed Tue Nov 17 02:38:52 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.908957. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25940 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98404 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77145 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43034 clock time +[SCHEDULER ITER26 LR0.010077696]: 560132 words processed Tue Nov 17 02:39:07 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.908535. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21194 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96840 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75766 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42111 clock time +[SCHEDULER ITER26 LR0.010077696]: 600118 words processed Tue Nov 17 02:39:22 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.906053. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19957 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96692 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42062 clock time +[SCHEDULER ITER26 LR0.010077696]: 640090 words processed Tue Nov 17 02:39:37 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.903747. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19791 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96533 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75110 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42072 clock time +[SCHEDULER ITER26 LR0.010077696]: 680075 words processed Tue Nov 17 02:39:52 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.902260. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20758 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96769 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75603 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42105 clock time +[SCHEDULER ITER26 LR0.010077696]: 720043 words processed Tue Nov 17 02:40:07 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.901047. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29414 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98784 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78360 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43269 clock time +[SCHEDULER ITER26 LR0.010077696]: 760012 words processed Tue Nov 17 02:40:22 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.899094. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34487 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00148 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80199 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43845 clock time +[SCHEDULER ITER26 LR0.010077696]: 800113 words processed Tue Nov 17 02:40:37 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.898673. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24692 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98485 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76166 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42941 clock time +[SCHEDULER ITER26 LR0.010077696]: 840089 words processed Tue Nov 17 02:40:52 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.897423. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19866 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96784 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74804 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42104 clock time +[SCHEDULER ITER26 LR0.010077696]: 880052 words processed Tue Nov 17 02:41:07 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-1.896938. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25747 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98148 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76873 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42776 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER26 LR0.010077696]: Displaying result: -[SCHEDULER ITER26 LR0.010077696]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.927606218813> <PPL_OOV 76.363419682672> <LOGP -1750309.5269079> +[SCHEDULER ITER26 LR0.010077696]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.751257092614> <PPL_OOV 78.864396682062> <LOGP -1763319.6982302> [SCHEDULER ITER26 LR0.010077696]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER26 LR0.010077696]: shuffling training file ===PEEK ON TEST 26=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER26 LR0.010077696]: 40087 words processed Mon Nov 16 21:48:26 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-2.106879. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69749 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77993 clock time +[SCHEDULER ITER26 LR0.010077696]: 40087 words processed Tue Nov 17 02:41:18 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-2.101186. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.75946 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79895 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER26 LR0.010077696]: Displaying result: -[SCHEDULER ITER26 LR0.010077696]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.9921749249> <PPL_OOV 128.49749497309> <LOGP -173836.18692982> +[SCHEDULER ITER26 LR0.010077696]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 145.13863540525> <PPL_OOV 127.0064555665> <LOGP -173418.36037405> [SCHEDULER ITER26 LR0.010077696]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 26=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER26 LR0.010077696]: 40095 words processed Mon Nov 16 21:48:39 2015. - [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-2.179472. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.73884 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80380 clock time +[SCHEDULER ITER26 LR0.010077696]: 40095 words processed Tue Nov 17 02:41:31 2015. + [SCHEDULER ITER26 LR0.010077696]: log prob per sample :-2.175300. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70468 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76903 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER26 LR0.010077696]: Displaying result: -[SCHEDULER ITER26 LR0.010077696]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 150.22176488574> <PPL_OOV 134.5693666043> <LOGP -157031.07232141> +[SCHEDULER ITER26 LR0.010077696]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.4917460737> <PPL_OOV 133.99975726498> <LOGP -156895.19189997> [SCHEDULER ITER26 LR0.010077696]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER26 LR0.010077696]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.26... @@ -5174,27 +5666,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1890181 +reading chunk 1 from 1893028 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1895031 +reading chunk 2 from 1897876 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48183648 +reading chunk 3 from 48153845 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48299860 +reading chunk 4 from 48270092 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94327301 +reading chunk 5 from 94304410 [SCHEDULER ITER27 LR0.0060466176]: preparing parameters end. [SCHEDULER ITER27 LR0.0060466176]: preparing layers... -(21:48:50 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:48:50 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:48:50 2015-11-16)[nerv] info: create layer: combinerL1 -(21:48:50 2015-11-16)[nerv] info: create layer: outputL -(21:48:50 2015-11-16)[nerv] info: create layer: softmaxL -(21:48:50 2015-11-16)[nerv] info: create layer: selectL1 +(02:41:42 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:41:42 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:41:42 2015-11-17)[nerv] info: create layer: combinerL1 +(02:41:42 2015-11-17)[nerv] info: create layer: outputL +(02:41:42 2015-11-17)[nerv] info: create layer: softmaxL +(02:41:42 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER27 LR0.0060466176]: preparing layers end. [SCHEDULER ITER27 LR0.0060466176]: Generate and initing TNN ... <input> selectL1 0 @@ -5222,164 +5714,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 27 LR 0.006047=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER27 LR0.0060466176]: 40092 words processed Mon Nov 16 21:49:05 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.918667. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29422 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00092 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78707 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44337 clock time -[SCHEDULER ITER27 LR0.0060466176]: 80099 words processed Mon Nov 16 21:49:20 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.914790. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35351 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99792 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83304 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44106 clock time -[SCHEDULER ITER27 LR0.0060466176]: 120004 words processed Mon Nov 16 21:49:35 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.908774. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23195 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97666 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77214 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43076 clock time -[SCHEDULER ITER27 LR0.0060466176]: 160114 words processed Mon Nov 16 21:49:50 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.909865. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39998 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01865 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83745 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45010 clock time -[SCHEDULER ITER27 LR0.0060466176]: 200066 words processed Mon Nov 16 21:50:05 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.909025. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31667 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99391 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80850 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43958 clock time -[SCHEDULER ITER27 LR0.0060466176]: 240045 words processed Mon Nov 16 21:50:20 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.905239. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26999 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98911 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78049 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43980 clock time -[SCHEDULER ITER27 LR0.0060466176]: 280057 words processed Mon Nov 16 21:50:35 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.901119. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36454 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01645 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81049 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45290 clock time -[SCHEDULER ITER27 LR0.0060466176]: 320106 words processed Mon Nov 16 21:50:50 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.899053. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32652 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00931 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79700 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45002 clock time -[SCHEDULER ITER27 LR0.0060466176]: 360024 words processed Mon Nov 16 21:51:05 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.897228. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19376 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96873 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76098 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42723 clock time -[SCHEDULER ITER27 LR0.0060466176]: 400089 words processed Mon Nov 16 21:51:20 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.894165. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25800 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99187 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43971 clock time -[SCHEDULER ITER27 LR0.0060466176]: 440067 words processed Mon Nov 16 21:51:35 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.894754. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28452 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98619 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80073 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43625 clock time -[SCHEDULER ITER27 LR0.0060466176]: 480051 words processed Mon Nov 16 21:51:50 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.894850. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23347 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97912 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77243 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43373 clock time -[SCHEDULER ITER27 LR0.0060466176]: 520140 words processed Mon Nov 16 21:52:05 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.894156. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33581 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01141 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80417 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45120 clock time -[SCHEDULER ITER27 LR0.0060466176]: 560132 words processed Mon Nov 16 21:52:20 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.893343. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20813 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97371 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76401 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43140 clock time -[SCHEDULER ITER27 LR0.0060466176]: 600118 words processed Mon Nov 16 21:52:35 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.890889. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29087 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99380 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79704 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43893 clock time -[SCHEDULER ITER27 LR0.0060466176]: 640090 words processed Mon Nov 16 21:52:50 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.888577. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21954 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97326 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77919 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42738 clock time -[SCHEDULER ITER27 LR0.0060466176]: 680075 words processed Mon Nov 16 21:53:05 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.887115. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20186 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97038 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76083 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43493 clock time -[SCHEDULER ITER27 LR0.0060466176]: 720043 words processed Mon Nov 16 21:53:20 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.885766. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18237 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97200 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74578 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43147 clock time -[SCHEDULER ITER27 LR0.0060466176]: 760012 words processed Mon Nov 16 21:53:35 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.883712. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20059 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97405 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75778 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42910 clock time -[SCHEDULER ITER27 LR0.0060466176]: 800113 words processed Mon Nov 16 21:53:50 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.883352. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23181 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98830 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76422 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43700 clock time -[SCHEDULER ITER27 LR0.0060466176]: 840089 words processed Mon Nov 16 21:54:04 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.882003. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17650 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96847 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74779 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42717 clock time -[SCHEDULER ITER27 LR0.0060466176]: 880052 words processed Mon Nov 16 21:54:19 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.881337. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21879 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97505 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77039 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43100 clock time +[SCHEDULER ITER27 LR0.0060466176]: 40092 words processed Tue Nov 17 02:41:56 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.928703. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33750 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99499 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80599 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43956 clock time +[SCHEDULER ITER27 LR0.0060466176]: 80099 words processed Tue Nov 17 02:42:11 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.925389. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26062 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97028 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78150 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42625 clock time +[SCHEDULER ITER27 LR0.0060466176]: 120004 words processed Tue Nov 17 02:42:26 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.920893. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22998 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95894 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77476 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42047 clock time +[SCHEDULER ITER27 LR0.0060466176]: 160114 words processed Tue Nov 17 02:42:41 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.922035. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25849 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97745 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77620 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42837 clock time +[SCHEDULER ITER27 LR0.0060466176]: 200066 words processed Tue Nov 17 02:42:56 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.921561. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21316 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95862 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76841 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42001 clock time +[SCHEDULER ITER27 LR0.0060466176]: 240045 words processed Tue Nov 17 02:43:11 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.917388. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22555 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77931 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42076 clock time +[SCHEDULER ITER27 LR0.0060466176]: 280057 words processed Tue Nov 17 02:43:26 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.913446. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26352 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97595 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78112 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42828 clock time +[SCHEDULER ITER27 LR0.0060466176]: 320106 words processed Tue Nov 17 02:43:41 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.911589. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26334 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97720 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78271 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42940 clock time +[SCHEDULER ITER27 LR0.0060466176]: 360024 words processed Tue Nov 17 02:43:56 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.909863. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26047 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78408 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42522 clock time +[SCHEDULER ITER27 LR0.0060466176]: 400089 words processed Tue Nov 17 02:44:11 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.906869. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28796 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98482 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79075 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43294 clock time +[SCHEDULER ITER27 LR0.0060466176]: 440067 words processed Tue Nov 17 02:44:26 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.907463. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20368 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95825 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76501 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41989 clock time +[SCHEDULER ITER27 LR0.0060466176]: 480051 words processed Tue Nov 17 02:44:41 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.907954. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95984 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76607 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41909 clock time +[SCHEDULER ITER27 LR0.0060466176]: 520140 words processed Tue Nov 17 02:44:56 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.907397. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30102 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98621 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79796 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43350 clock time +[SCHEDULER ITER27 LR0.0060466176]: 560132 words processed Tue Nov 17 02:45:11 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.906986. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28358 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97735 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79206 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42844 clock time +[SCHEDULER ITER27 LR0.0060466176]: 600118 words processed Tue Nov 17 02:45:26 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.904509. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23353 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77820 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42521 clock time +[SCHEDULER ITER27 LR0.0060466176]: 640090 words processed Tue Nov 17 02:45:41 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.902200. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35502 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99495 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82236 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44239 clock time +[SCHEDULER ITER27 LR0.0060466176]: 680075 words processed Tue Nov 17 02:45:56 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.900713. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23826 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96820 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77828 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42484 clock time +[SCHEDULER ITER27 LR0.0060466176]: 720043 words processed Tue Nov 17 02:46:11 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.899495. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24761 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96609 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78338 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42448 clock time +[SCHEDULER ITER27 LR0.0060466176]: 760012 words processed Tue Nov 17 02:46:26 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.897538. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20772 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95950 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76350 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42087 clock time +[SCHEDULER ITER27 LR0.0060466176]: 800113 words processed Tue Nov 17 02:46:41 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.897123. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34334 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99453 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81319 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43889 clock time +[SCHEDULER ITER27 LR0.0060466176]: 840089 words processed Tue Nov 17 02:46:56 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.895873. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97044 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78450 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42452 clock time +[SCHEDULER ITER27 LR0.0060466176]: 880052 words processed Tue Nov 17 02:47:11 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-1.895386. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25367 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97014 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78390 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42554 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER27 LR0.0060466176]: Displaying result: -[SCHEDULER ITER27 LR0.0060466176]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.611310989666> <PPL_OOV 76.082052253075> <LOGP -1748819.2565264> +[SCHEDULER ITER27 LR0.0060466176]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.434859688779> <PPL_OOV 78.582944474359> <LOGP -1761876.3368721> [SCHEDULER ITER27 LR0.0060466176]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER27 LR0.0060466176]: shuffling training file ===PEEK ON TEST 27=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER27 LR0.0060466176]: 40087 words processed Mon Nov 16 21:54:31 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-2.106093. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.75428 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81034 clock time +[SCHEDULER ITER27 LR0.0060466176]: 40087 words processed Tue Nov 17 02:47:22 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-2.100389. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70063 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77711 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER27 LR0.0060466176]: Displaying result: -[SCHEDULER ITER27 LR0.0060466176]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.66521733895> <PPL_OOV 128.2457381726> <LOGP -173765.97968168> +[SCHEDULER ITER27 LR0.0060466176]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.84974138665> <PPL_OOV 126.75856299896> <LOGP -173348.41942903> [SCHEDULER ITER27 LR0.0060466176]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 27=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER27 LR0.0060466176]: 40095 words processed Mon Nov 16 21:54:45 2015. - [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-2.178726. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70404 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78149 clock time +[SCHEDULER ITER27 LR0.0060466176]: 40095 words processed Tue Nov 17 02:47:35 2015. + [SCHEDULER ITER27 LR0.0060466176]: log prob per sample :-2.174504. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69769 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77520 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER27 LR0.0060466176]: Displaying result: -[SCHEDULER ITER27 LR0.0060466176]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.878892758> <PPL_OOV 134.29600411411> <LOGP -156965.93357189> +[SCHEDULER ITER27 LR0.0060466176]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.14618288831> <PPL_OOV 133.69469792076> <LOGP -156822.1822343> [SCHEDULER ITER27 LR0.0060466176]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER27 LR0.0060466176]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.27... @@ -5388,27 +5880,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46027363 +reading chunk 1 from 46034294 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46032213 +reading chunk 2 from 46039142 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92320760 +reading chunk 3 from 92295019 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92436970 +reading chunk 4 from 92411266 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94327115 +reading chunk 5 from 94304283 [SCHEDULER ITER28 LR0.00362797056]: preparing parameters end. [SCHEDULER ITER28 LR0.00362797056]: preparing layers... -(21:54:57 2015-11-16)[nerv] info: create layer: recurrentL1 -(21:54:57 2015-11-16)[nerv] info: create layer: sigmoidL1 -(21:54:57 2015-11-16)[nerv] info: create layer: combinerL1 -(21:54:57 2015-11-16)[nerv] info: create layer: outputL -(21:54:57 2015-11-16)[nerv] info: create layer: softmaxL -(21:54:57 2015-11-16)[nerv] info: create layer: selectL1 +(02:47:46 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:47:46 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:47:46 2015-11-17)[nerv] info: create layer: combinerL1 +(02:47:46 2015-11-17)[nerv] info: create layer: outputL +(02:47:46 2015-11-17)[nerv] info: create layer: softmaxL +(02:47:46 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER28 LR0.00362797056]: preparing layers end. [SCHEDULER ITER28 LR0.00362797056]: Generate and initing TNN ... <input> selectL1 0 @@ -5436,164 +5928,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 28 LR 0.003628=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER28 LR0.00362797056]: 40092 words processed Mon Nov 16 21:55:12 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.917576. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37494 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98013 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85753 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45018 clock time -[SCHEDULER ITER28 LR0.00362797056]: 80099 words processed Mon Nov 16 21:55:27 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.913671. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31013 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95996 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44370 clock time -[SCHEDULER ITER28 LR0.00362797056]: 120004 words processed Mon Nov 16 21:55:42 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.907684. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34703 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96673 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84843 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44506 clock time -[SCHEDULER ITER28 LR0.00362797056]: 160114 words processed Mon Nov 16 21:55:57 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.908779. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33791 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97029 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84254 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44572 clock time -[SCHEDULER ITER28 LR0.00362797056]: 200066 words processed Mon Nov 16 21:56:12 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.907960. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17533 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92923 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77688 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42957 clock time -[SCHEDULER ITER28 LR0.00362797056]: 240045 words processed Mon Nov 16 21:56:26 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.904162. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15009 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92707 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76067 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42718 clock time -[SCHEDULER ITER28 LR0.00362797056]: 280057 words processed Mon Nov 16 21:56:41 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.900061. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20334 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94680 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77593 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43721 clock time -[SCHEDULER ITER28 LR0.00362797056]: 320106 words processed Mon Nov 16 21:56:56 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.898018. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19358 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94287 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77484 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43528 clock time -[SCHEDULER ITER28 LR0.00362797056]: 360024 words processed Mon Nov 16 21:57:11 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.896201. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22177 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93655 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78661 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44282 clock time -[SCHEDULER ITER28 LR0.00362797056]: 400089 words processed Mon Nov 16 21:57:26 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.893131. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27341 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96345 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80394 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44369 clock time -[SCHEDULER ITER28 LR0.00362797056]: 440067 words processed Mon Nov 16 21:57:41 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.893731. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15873 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92865 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76925 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42799 clock time -[SCHEDULER ITER28 LR0.00362797056]: 480051 words processed Mon Nov 16 21:57:55 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.893827. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14938 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92560 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76483 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42915 clock time -[SCHEDULER ITER28 LR0.00362797056]: 520140 words processed Mon Nov 16 21:58:10 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.893133. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19809 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94384 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77838 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43647 clock time -[SCHEDULER ITER28 LR0.00362797056]: 560132 words processed Mon Nov 16 21:58:24 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.892323. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.14647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92543 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76529 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42637 clock time -[SCHEDULER ITER28 LR0.00362797056]: 600118 words processed Mon Nov 16 21:58:39 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.889876. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24640 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94950 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80546 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43907 clock time -[SCHEDULER ITER28 LR0.00362797056]: 640090 words processed Mon Nov 16 21:58:54 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.887566. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24333 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95018 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80145 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43933 clock time -[SCHEDULER ITER28 LR0.00362797056]: 680075 words processed Mon Nov 16 21:59:09 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.886103. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22660 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93899 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80709 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43428 clock time -[SCHEDULER ITER28 LR0.00362797056]: 720043 words processed Mon Nov 16 21:59:24 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.884762. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20967 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93770 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79890 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42994 clock time -[SCHEDULER ITER28 LR0.00362797056]: 760012 words processed Mon Nov 16 21:59:39 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.882705. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34439 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96422 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85748 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44249 clock time -[SCHEDULER ITER28 LR0.00362797056]: 800113 words processed Mon Nov 16 21:59:54 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.882343. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31165 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96704 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83053 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44860 clock time -[SCHEDULER ITER28 LR0.00362797056]: 840089 words processed Mon Nov 16 22:00:09 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.880989. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29123 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95247 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84056 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43621 clock time -[SCHEDULER ITER28 LR0.00362797056]: 880052 words processed Mon Nov 16 22:00:24 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.880322. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29170 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95471 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83438 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43746 clock time +[SCHEDULER ITER28 LR0.00362797056]: 40092 words processed Tue Nov 17 02:48:01 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.927532. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34231 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00674 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79762 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43937 clock time +[SCHEDULER ITER28 LR0.00362797056]: 80099 words processed Tue Nov 17 02:48:16 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.924242. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26584 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77423 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42657 clock time +[SCHEDULER ITER28 LR0.00362797056]: 120004 words processed Tue Nov 17 02:48:31 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.919807. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30861 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99103 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79079 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43114 clock time +[SCHEDULER ITER28 LR0.00362797056]: 160114 words processed Tue Nov 17 02:48:46 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.920948. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.40793 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.02137 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82324 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44675 clock time +[SCHEDULER ITER28 LR0.00362797056]: 200066 words processed Tue Nov 17 02:49:01 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.920510. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28854 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98477 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78655 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42864 clock time +[SCHEDULER ITER28 LR0.00362797056]: 240045 words processed Tue Nov 17 02:49:16 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.916340. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25393 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97873 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77432 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42618 clock time +[SCHEDULER ITER28 LR0.00362797056]: 280057 words processed Tue Nov 17 02:49:31 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.912409. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26830 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98697 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77609 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43121 clock time +[SCHEDULER ITER28 LR0.00362797056]: 320106 words processed Tue Nov 17 02:49:46 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.910574. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27498 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98970 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77673 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43303 clock time +[SCHEDULER ITER28 LR0.00362797056]: 360024 words processed Tue Nov 17 02:50:01 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.908850. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22678 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97153 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76735 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42291 clock time +[SCHEDULER ITER28 LR0.00362797056]: 400089 words processed Tue Nov 17 02:50:16 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.905859. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28973 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99486 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78305 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43455 clock time +[SCHEDULER ITER28 LR0.00362797056]: 440067 words processed Tue Nov 17 02:50:31 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.906460. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20430 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97006 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75490 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42183 clock time +[SCHEDULER ITER28 LR0.00362797056]: 480051 words processed Tue Nov 17 02:50:46 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.906956. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21321 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97149 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76167 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42170 clock time +[SCHEDULER ITER28 LR0.00362797056]: 520140 words processed Tue Nov 17 02:51:01 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.906400. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32795 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00272 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79749 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44005 clock time +[SCHEDULER ITER28 LR0.00362797056]: 560132 words processed Tue Nov 17 02:51:16 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.905997. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23221 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97764 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76683 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42353 clock time +[SCHEDULER ITER28 LR0.00362797056]: 600118 words processed Tue Nov 17 02:51:31 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.903523. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21445 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97178 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75911 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42355 clock time +[SCHEDULER ITER28 LR0.00362797056]: 640090 words processed Tue Nov 17 02:51:46 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.901214. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24051 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97828 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77055 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42471 clock time +[SCHEDULER ITER28 LR0.00362797056]: 680075 words processed Tue Nov 17 02:52:01 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.899727. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33866 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99900 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80458 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43604 clock time +[SCHEDULER ITER28 LR0.00362797056]: 720043 words processed Tue Nov 17 02:52:16 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.898506. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20307 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97020 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75335 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42146 clock time +[SCHEDULER ITER28 LR0.00362797056]: 760012 words processed Tue Nov 17 02:52:31 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.896546. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26163 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98292 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77632 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42774 clock time +[SCHEDULER ITER28 LR0.00362797056]: 800113 words processed Tue Nov 17 02:52:46 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.896134. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28570 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99602 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78017 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43537 clock time +[SCHEDULER ITER28 LR0.00362797056]: 840089 words processed Tue Nov 17 02:53:01 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.894884. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28293 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78332 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42941 clock time +[SCHEDULER ITER28 LR0.00362797056]: 880052 words processed Tue Nov 17 02:53:16 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-1.894395. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30700 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99105 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79397 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43283 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER28 LR0.00362797056]: Displaying result: -[SCHEDULER ITER28 LR0.00362797056]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.411191347458> <PPL_OOV 75.904331078529> <LOGP -1747875.1087491> +[SCHEDULER ITER28 LR0.00362797056]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.233576555989> <PPL_OOV 78.404021889721> <LOGP -1760956.0843063> [SCHEDULER ITER28 LR0.00362797056]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER28 LR0.00362797056]: shuffling training file ===PEEK ON TEST 28=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER28 LR0.00362797056]: 40087 words processed Mon Nov 16 22:00:36 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-2.105555. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77824 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82478 clock time +[SCHEDULER ITER28 LR0.00362797056]: 40087 words processed Tue Nov 17 02:53:27 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-2.099856. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.78814 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82154 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER28 LR0.00362797056]: Displaying result: -[SCHEDULER ITER28 LR0.00362797056]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.48378228665> <PPL_OOV 128.07182003966> <LOGP -173717.39871181> +[SCHEDULER ITER28 LR0.00362797056]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.69837034103> <PPL_OOV 126.58856655114> <LOGP -173300.37715782> [SCHEDULER ITER28 LR0.00362797056]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 28=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER28 LR0.00362797056]: 40095 words processed Mon Nov 16 22:00:49 2015. - [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-2.178284. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.78609 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83157 clock time +[SCHEDULER ITER28 LR0.00362797056]: 40095 words processed Tue Nov 17 02:53:40 2015. + [SCHEDULER ITER28 LR0.00362797056]: log prob per sample :-2.173991. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69967 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76790 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER28 LR0.00362797056]: Displaying result: -[SCHEDULER ITER28 LR0.00362797056]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.70634636213> <PPL_OOV 134.12966304144> <LOGP -156926.23172345> +[SCHEDULER ITER28 LR0.00362797056]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 148.96680910385> <PPL_OOV 133.50103116837> <LOGP -156775.74558779> [SCHEDULER ITER28 LR0.00362797056]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER28 LR0.00362797056]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.28... @@ -5602,27 +6094,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1890155 +reading chunk 1 from 1893023 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1895005 +reading chunk 2 from 1897872 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48183478 +reading chunk 3 from 48153679 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48299687 +reading chunk 4 from 48269927 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94326991 +reading chunk 5 from 94304217 [SCHEDULER ITER29 LR0.002176782336]: preparing parameters end. [SCHEDULER ITER29 LR0.002176782336]: preparing layers... -(22:01:00 2015-11-16)[nerv] info: create layer: recurrentL1 -(22:01:00 2015-11-16)[nerv] info: create layer: sigmoidL1 -(22:01:00 2015-11-16)[nerv] info: create layer: combinerL1 -(22:01:00 2015-11-16)[nerv] info: create layer: outputL -(22:01:00 2015-11-16)[nerv] info: create layer: softmaxL -(22:01:00 2015-11-16)[nerv] info: create layer: selectL1 +(02:53:51 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:53:51 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:53:51 2015-11-17)[nerv] info: create layer: combinerL1 +(02:53:51 2015-11-17)[nerv] info: create layer: outputL +(02:53:51 2015-11-17)[nerv] info: create layer: softmaxL +(02:53:51 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER29 LR0.002176782336]: preparing layers end. [SCHEDULER ITER29 LR0.002176782336]: Generate and initing TNN ... <input> selectL1 0 @@ -5650,164 +6142,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 29 LR 0.002177=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER29 LR0.002176782336]: 40092 words processed Mon Nov 16 22:01:14 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.916882. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21352 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97794 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75575 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43721 clock time -[SCHEDULER ITER29 LR0.002176782336]: 80099 words processed Mon Nov 16 22:01:29 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.912963. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31080 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98620 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81424 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43868 clock time -[SCHEDULER ITER29 LR0.002176782336]: 120004 words processed Mon Nov 16 22:01:44 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.906983. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20362 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96404 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76834 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42775 clock time -[SCHEDULER ITER29 LR0.002176782336]: 160114 words processed Mon Nov 16 22:01:59 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.908085. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30572 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99605 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80354 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44404 clock time -[SCHEDULER ITER29 LR0.002176782336]: 200066 words processed Mon Nov 16 22:02:13 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.907277. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17040 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95668 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75424 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42687 clock time -[SCHEDULER ITER29 LR0.002176782336]: 240045 words processed Mon Nov 16 22:02:28 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.903480. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19665 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96169 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76820 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42899 clock time -[SCHEDULER ITER29 LR0.002176782336]: 280057 words processed Mon Nov 16 22:02:43 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.899390. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00178 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83534 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44724 clock time -[SCHEDULER ITER29 LR0.002176782336]: 320106 words processed Mon Nov 16 22:02:58 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.897358. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37460 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00756 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83764 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45078 clock time -[SCHEDULER ITER29 LR0.002176782336]: 360024 words processed Mon Nov 16 22:03:12 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.895550. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16885 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95677 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75628 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42850 clock time -[SCHEDULER ITER29 LR0.002176782336]: 400089 words processed Mon Nov 16 22:03:27 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.892479. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26094 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98884 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77536 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44776 clock time -[SCHEDULER ITER29 LR0.002176782336]: 440067 words processed Mon Nov 16 22:03:41 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.893085. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16530 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95764 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75238 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42697 clock time -[SCHEDULER ITER29 LR0.002176782336]: 480051 words processed Mon Nov 16 22:03:55 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.893185. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16141 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95691 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75050 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42806 clock time -[SCHEDULER ITER29 LR0.002176782336]: 520140 words processed Mon Nov 16 22:04:10 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.892489. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21031 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97574 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76257 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43645 clock time -[SCHEDULER ITER29 LR0.002176782336]: 560132 words processed Mon Nov 16 22:04:24 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.891675. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15918 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95901 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74692 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42864 clock time -[SCHEDULER ITER29 LR0.002176782336]: 600118 words processed Mon Nov 16 22:04:38 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.889232. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16672 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95919 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75390 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42737 clock time -[SCHEDULER ITER29 LR0.002176782336]: 640090 words processed Mon Nov 16 22:04:53 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.886925. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19147 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96142 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76958 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42897 clock time -[SCHEDULER ITER29 LR0.002176782336]: 680075 words processed Mon Nov 16 22:05:08 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.885457. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23279 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96863 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79242 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43133 clock time -[SCHEDULER ITER29 LR0.002176782336]: 720043 words processed Mon Nov 16 22:05:23 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.884124. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23083 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96896 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79257 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42977 clock time -[SCHEDULER ITER29 LR0.002176782336]: 760012 words processed Mon Nov 16 22:05:38 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.882065. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99360 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81520 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44613 clock time -[SCHEDULER ITER29 LR0.002176782336]: 800113 words processed Mon Nov 16 22:05:53 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.881700. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31276 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00368 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79691 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44927 clock time -[SCHEDULER ITER29 LR0.002176782336]: 840089 words processed Mon Nov 16 22:06:08 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.880344. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30257 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99428 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79632 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44632 clock time -[SCHEDULER ITER29 LR0.002176782336]: 880052 words processed Mon Nov 16 22:06:23 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.879675. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28460 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98816 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79735 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44172 clock time +[SCHEDULER ITER29 LR0.002176782336]: 40092 words processed Tue Nov 17 02:54:05 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.926804. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27248 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98345 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77122 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43456 clock time +[SCHEDULER ITER29 LR0.002176782336]: 80099 words processed Tue Nov 17 02:54:20 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.923520. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20632 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95859 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75464 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42067 clock time +[SCHEDULER ITER29 LR0.002176782336]: 120004 words processed Tue Nov 17 02:54:35 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.919123. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23342 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95793 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76997 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42236 clock time +[SCHEDULER ITER29 LR0.002176782336]: 160114 words processed Tue Nov 17 02:54:50 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.920275. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30103 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98986 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78615 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43429 clock time +[SCHEDULER ITER29 LR0.002176782336]: 200066 words processed Tue Nov 17 02:55:05 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.919859. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23302 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96382 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76885 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42441 clock time +[SCHEDULER ITER29 LR0.002176782336]: 240045 words processed Tue Nov 17 02:55:20 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.915689. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20595 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95842 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75896 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42134 clock time +[SCHEDULER ITER29 LR0.002176782336]: 280057 words processed Tue Nov 17 02:55:35 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.911762. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98107 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78275 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43299 clock time +[SCHEDULER ITER29 LR0.002176782336]: 320106 words processed Tue Nov 17 02:55:50 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.909944. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24132 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97236 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76708 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42912 clock time +[SCHEDULER ITER29 LR0.002176782336]: 360024 words processed Tue Nov 17 02:56:05 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.908221. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19949 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95635 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75753 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42132 clock time +[SCHEDULER ITER29 LR0.002176782336]: 400089 words processed Tue Nov 17 02:56:20 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.905228. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25159 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97580 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77227 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43160 clock time +[SCHEDULER ITER29 LR0.002176782336]: 440067 words processed Tue Nov 17 02:56:35 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.905830. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19959 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95565 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76024 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42101 clock time +[SCHEDULER ITER29 LR0.002176782336]: 480051 words processed Tue Nov 17 02:56:50 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.906331. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22012 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96398 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76480 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42342 clock time +[SCHEDULER ITER29 LR0.002176782336]: 520140 words processed Tue Nov 17 02:57:05 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.905774. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25240 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97564 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77114 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43086 clock time +[SCHEDULER ITER29 LR0.002176782336]: 560132 words processed Tue Nov 17 02:57:20 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.905376. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22671 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96372 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76859 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42522 clock time +[SCHEDULER ITER29 LR0.002176782336]: 600118 words processed Tue Nov 17 02:57:35 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.902905. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19201 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95582 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75560 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41996 clock time +[SCHEDULER ITER29 LR0.002176782336]: 640090 words processed Tue Nov 17 02:57:50 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.900596. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25047 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96933 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77735 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42674 clock time +[SCHEDULER ITER29 LR0.002176782336]: 680075 words processed Tue Nov 17 02:58:05 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.899109. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97812 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79342 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43254 clock time +[SCHEDULER ITER29 LR0.002176782336]: 720043 words processed Tue Nov 17 02:58:20 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.897886. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26705 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97345 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78078 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43116 clock time +[SCHEDULER ITER29 LR0.002176782336]: 760012 words processed Tue Nov 17 02:58:35 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.895924. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18244 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95299 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74974 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42067 clock time +[SCHEDULER ITER29 LR0.002176782336]: 800113 words processed Tue Nov 17 02:58:50 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.895513. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25828 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97812 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77331 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43061 clock time +[SCHEDULER ITER29 LR0.002176782336]: 840089 words processed Tue Nov 17 02:59:05 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.894262. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27907 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79075 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43221 clock time +[SCHEDULER ITER29 LR0.002176782336]: 880052 words processed Tue Nov 17 02:59:20 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-1.893773. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22267 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96156 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76727 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42522 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER29 LR0.002176782336]: Displaying result: -[SCHEDULER ITER29 LR0.002176782336]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.283901114873> <PPL_OOV 75.791502900064> <LOGP -1747274.5585906> +[SCHEDULER ITER29 LR0.002176782336]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.107347759274> <PPL_OOV 78.291743027047> <LOGP -1760377.5273718> [SCHEDULER ITER29 LR0.002176782336]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER29 LR0.002176782336]: shuffling training file ===PEEK ON TEST 29=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER29 LR0.002176782336]: 40087 words processed Mon Nov 16 22:06:34 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-2.105165. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68228 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76352 clock time +[SCHEDULER ITER29 LR0.002176782336]: 40087 words processed Tue Nov 17 02:59:31 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-2.099495. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70427 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77050 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER29 LR0.002176782336]: Displaying result: -[SCHEDULER ITER29 LR0.002176782336]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.37737062437> <PPL_OOV 127.94693234979> <LOGP -173682.47282062> +[SCHEDULER ITER29 LR0.002176782336]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.62131391186> <PPL_OOV 126.47430662671> <LOGP -173268.05017784> [SCHEDULER ITER29 LR0.002176782336]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 29=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER29 LR0.002176782336]: 40095 words processed Mon Nov 16 22:06:48 2015. - [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-2.178008. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.82802 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86294 clock time +[SCHEDULER ITER29 LR0.002176782336]: 40095 words processed Tue Nov 17 02:59:44 2015. + [SCHEDULER ITER29 LR0.002176782336]: log prob per sample :-2.173646. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72996 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78708 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER29 LR0.002176782336]: Displaying result: -[SCHEDULER ITER29 LR0.002176782336]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.61642875592> <PPL_OOV 134.02237108393> <LOGP -156900.59743237> +[SCHEDULER ITER29 LR0.002176782336]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 148.87647174753> <PPL_OOV 133.37933819971> <LOGP -156746.53205356> [SCHEDULER ITER29 LR0.002176782336]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER29 LR0.002176782336]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.29... @@ -5816,27 +6308,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46027283 +reading chunk 1 from 46034249 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46032133 +reading chunk 2 from 46039098 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92320596 +reading chunk 3 from 92294894 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92436806 +reading chunk 4 from 92411143 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94326944 +reading chunk 5 from 94304171 [SCHEDULER ITER30 LR0.0013060694016]: preparing parameters end. [SCHEDULER ITER30 LR0.0013060694016]: preparing layers... -(22:07:00 2015-11-16)[nerv] info: create layer: recurrentL1 -(22:07:00 2015-11-16)[nerv] info: create layer: sigmoidL1 -(22:07:00 2015-11-16)[nerv] info: create layer: combinerL1 -(22:07:00 2015-11-16)[nerv] info: create layer: outputL -(22:07:00 2015-11-16)[nerv] info: create layer: softmaxL -(22:07:00 2015-11-16)[nerv] info: create layer: selectL1 +(02:59:56 2015-11-17)[nerv] info: create layer: recurrentL1 +(02:59:56 2015-11-17)[nerv] info: create layer: sigmoidL1 +(02:59:56 2015-11-17)[nerv] info: create layer: combinerL1 +(02:59:56 2015-11-17)[nerv] info: create layer: outputL +(02:59:56 2015-11-17)[nerv] info: create layer: softmaxL +(02:59:56 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER30 LR0.0013060694016]: preparing layers end. [SCHEDULER ITER30 LR0.0013060694016]: Generate and initing TNN ... <input> selectL1 0 @@ -5864,164 +6356,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 30 LR 0.001306=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER30 LR0.0013060694016]: 40092 words processed Mon Nov 16 22:07:15 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.916462. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34544 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96984 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86168 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44426 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 80099 words processed Mon Nov 16 22:07:30 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.912531. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27213 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94216 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84102 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43109 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 120004 words processed Mon Nov 16 22:07:45 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.906550. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23417 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93593 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81632 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43257 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 160114 words processed Mon Nov 16 22:08:00 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.907654. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94462 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79974 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43743 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 200066 words processed Mon Nov 16 22:08:15 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.906847. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25451 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94013 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83491 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43064 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 240045 words processed Mon Nov 16 22:08:30 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.903052. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20747 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93575 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79404 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43532 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 280057 words processed Mon Nov 16 22:08:45 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.898971. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27105 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95182 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82865 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43897 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 320106 words processed Mon Nov 16 22:09:00 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.896943. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34224 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96947 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85967 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44502 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 360024 words processed Mon Nov 16 22:09:15 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.895141. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95999 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85005 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44759 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 400089 words processed Mon Nov 16 22:09:30 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.892069. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30006 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96226 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84317 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44114 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 440067 words processed Mon Nov 16 22:09:45 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.892680. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19921 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93372 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79921 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43055 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 480051 words processed Mon Nov 16 22:10:00 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.892783. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94176 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80586 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43038 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 520140 words processed Mon Nov 16 22:10:15 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.892086. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.40756 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98216 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.89247 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45274 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 560132 words processed Mon Nov 16 22:10:30 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.891271. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36687 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96626 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.88119 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44352 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 600118 words processed Mon Nov 16 22:10:45 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.888832. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27742 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94694 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84809 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43235 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 640090 words processed Mon Nov 16 22:11:00 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.886525. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21550 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93323 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81656 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42993 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 680075 words processed Mon Nov 16 22:11:15 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.885052. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23887 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94026 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82176 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43158 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 720043 words processed Mon Nov 16 22:11:30 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.883724. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20587 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93406 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80840 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42978 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 760012 words processed Mon Nov 16 22:11:45 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.881662. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22949 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93788 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82359 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42909 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 800113 words processed Mon Nov 16 22:12:00 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.881296. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27679 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95527 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82797 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44197 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 840089 words processed Mon Nov 16 22:12:15 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.879938. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19541 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93559 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79710 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42903 clock time -[SCHEDULER ITER30 LR0.0013060694016]: 880052 words processed Mon Nov 16 22:12:30 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.879269. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24382 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94244 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82279 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43304 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 40092 words processed Tue Nov 17 03:00:11 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.926353. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29578 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00309 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76685 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43358 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 80099 words processed Tue Nov 17 03:00:26 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.923072. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20520 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97125 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73944 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42110 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 120004 words processed Tue Nov 17 03:00:41 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.918701. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20066 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97144 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73638 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41979 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 160114 words processed Tue Nov 17 03:00:56 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.919857. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38621 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.02250 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80262 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44486 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 200066 words processed Tue Nov 17 03:01:11 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.919459. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30458 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99556 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77797 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43214 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 240045 words processed Tue Nov 17 03:01:26 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.915288. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98893 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75720 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43198 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 280057 words processed Tue Nov 17 03:01:41 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.911363. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32270 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00751 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77703 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43697 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 320106 words processed Tue Nov 17 03:01:56 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.909556. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27435 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99491 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76453 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43137 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 360024 words processed Tue Nov 17 03:02:11 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.907832. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34022 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00410 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79273 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43719 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 400089 words processed Tue Nov 17 03:02:26 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.904837. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31665 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00956 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77781 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43848 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 440067 words processed Tue Nov 17 03:02:41 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.905440. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29028 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99421 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77467 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42947 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 480051 words processed Tue Nov 17 03:02:56 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.905945. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19277 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97187 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73723 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42157 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 520140 words processed Tue Nov 17 03:03:11 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.905387. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24924 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99111 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75478 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42938 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 560132 words processed Tue Nov 17 03:03:26 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.904991. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19247 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97203 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73966 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42057 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 600118 words processed Tue Nov 17 03:03:41 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.902521. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20645 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97575 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74441 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42251 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 640090 words processed Tue Nov 17 03:03:56 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.900211. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28761 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99519 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77220 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43357 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 680075 words processed Tue Nov 17 03:04:11 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.898724. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34316 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00309 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79311 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43886 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 720043 words processed Tue Nov 17 03:04:26 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.897501. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34610 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00513 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79882 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43632 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 760012 words processed Tue Nov 17 03:04:41 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.895537. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20355 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97550 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73981 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42201 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 800113 words processed Tue Nov 17 03:04:56 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.895126. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23987 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98981 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74942 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42960 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 840089 words processed Tue Nov 17 03:05:11 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.893875. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21999 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97886 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74974 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42419 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 880052 words processed Tue Nov 17 03:05:26 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-1.893383. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33073 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00486 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78516 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43468 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER30 LR0.0013060694016]: Displaying result: -[SCHEDULER ITER30 LR0.0013060694016]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.204123746308> <PPL_OOV 75.72049201999> <LOGP -1746896.1306508> +[SCHEDULER ITER30 LR0.0013060694016]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 86.028294764601> <PPL_OOV 78.221628504544> <LOGP -1760015.8162639> [SCHEDULER ITER30 LR0.0013060694016]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER30 LR0.0013060694016]: shuffling training file ===PEEK ON TEST 30=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER30 LR0.0013060694016]: 40087 words processed Mon Nov 16 22:12:41 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-2.104864. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77509 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83242 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 40087 words processed Tue Nov 17 03:05:37 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-2.099248. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69638 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75793 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER30 LR0.0013060694016]: Displaying result: -[SCHEDULER ITER30 LR0.0013060694016]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.28149529838> <PPL_OOV 127.85129289029> <LOGP -173655.70338512> +[SCHEDULER ITER30 LR0.0013060694016]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.5740542712> <PPL_OOV 126.39864807499> <LOGP -173246.62841237> [SCHEDULER ITER30 LR0.0013060694016]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 30=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER30 LR0.0013060694016]: 40095 words processed Mon Nov 16 22:12:54 2015. - [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-2.177800. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.74251 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81400 clock time +[SCHEDULER ITER30 LR0.0013060694016]: 40095 words processed Tue Nov 17 03:05:50 2015. + [SCHEDULER ITER30 LR0.0013060694016]: log prob per sample :-2.173404. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.67502 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74523 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER30 LR0.0013060694016]: Displaying result: -[SCHEDULER ITER30 LR0.0013060694016]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.53856157397> <PPL_OOV 133.94261488579> <LOGP -156881.52870888> +[SCHEDULER ITER30 LR0.0013060694016]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 148.81926533953> <PPL_OOV 133.29866678303> <LOGP -156727.15143141> [SCHEDULER ITER30 LR0.0013060694016]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER30 LR0.0013060694016]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.30... @@ -6030,27 +6522,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1890138 +reading chunk 1 from 1893030 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1894988 +reading chunk 2 from 1897879 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48183411 +reading chunk 3 from 48153629 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48299621 +reading chunk 4 from 48269877 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94326897 +reading chunk 5 from 94304103 [SCHEDULER ITER31 LR0.00078364164096]: preparing parameters end. [SCHEDULER ITER31 LR0.00078364164096]: preparing layers... -(22:13:05 2015-11-16)[nerv] info: create layer: recurrentL1 -(22:13:05 2015-11-16)[nerv] info: create layer: sigmoidL1 -(22:13:05 2015-11-16)[nerv] info: create layer: combinerL1 -(22:13:05 2015-11-16)[nerv] info: create layer: outputL -(22:13:05 2015-11-16)[nerv] info: create layer: softmaxL -(22:13:05 2015-11-16)[nerv] info: create layer: selectL1 +(03:06:01 2015-11-17)[nerv] info: create layer: recurrentL1 +(03:06:01 2015-11-17)[nerv] info: create layer: sigmoidL1 +(03:06:01 2015-11-17)[nerv] info: create layer: combinerL1 +(03:06:01 2015-11-17)[nerv] info: create layer: outputL +(03:06:01 2015-11-17)[nerv] info: create layer: softmaxL +(03:06:01 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER31 LR0.00078364164096]: preparing layers end. [SCHEDULER ITER31 LR0.00078364164096]: Generate and initing TNN ... <input> selectL1 0 @@ -6078,164 +6570,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 31 LR 0.000784=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER31 LR0.00078364164096]: 40092 words processed Mon Nov 16 22:13:20 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.916210. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.40506 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01801 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85923 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45082 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 80099 words processed Mon Nov 16 22:13:35 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.912266. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28071 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98005 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80760 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43510 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 120004 words processed Mon Nov 16 22:13:50 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.906285. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25788 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97473 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79618 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43233 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 160114 words processed Mon Nov 16 22:14:05 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.907390. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26915 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98494 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79994 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43797 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 200066 words processed Mon Nov 16 22:14:20 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.906581. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19876 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96263 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77290 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42787 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 240045 words processed Mon Nov 16 22:14:35 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.902782. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20136 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96436 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77429 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42846 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 280057 words processed Mon Nov 16 22:14:50 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.898708. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.40144 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01699 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84361 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45199 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 320106 words processed Mon Nov 16 22:15:05 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.896684. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29865 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99708 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80316 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44505 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 360024 words processed Mon Nov 16 22:15:20 2015. +[SCHEDULER ITER31 LR0.00078364164096]: 40092 words processed Tue Nov 17 03:06:16 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.926079. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37107 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99890 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82190 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44072 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 80099 words processed Tue Nov 17 03:06:31 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.922803. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31134 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97913 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80356 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43153 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 120004 words processed Tue Nov 17 03:06:46 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.918438. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34457 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98541 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80956 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43567 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 160114 words processed Tue Nov 17 03:07:01 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.919593. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36112 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99914 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81623 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43992 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 200066 words processed Tue Nov 17 03:07:16 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.919209. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27147 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97083 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78808 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42824 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 240045 words processed Tue Nov 17 03:07:31 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.915037. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29722 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97811 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79786 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43142 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 280057 words processed Tue Nov 17 03:07:46 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.911113. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38977 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00405 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82949 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44412 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 320106 words processed Tue Nov 17 03:08:01 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.909313. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33405 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99531 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80731 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43590 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 360024 words processed Tue Nov 17 03:08:16 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.907589. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31706 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98374 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80608 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43378 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 400089 words processed Tue Nov 17 03:08:31 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.904592. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35210 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00199 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81229 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44025 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 440067 words processed Tue Nov 17 03:08:46 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.905194. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21590 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96014 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76916 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42266 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 480051 words processed Tue Nov 17 03:09:01 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.905704. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21467 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95967 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77061 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42287 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 520140 words processed Tue Nov 17 03:09:16 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.905144. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27335 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97718 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78837 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43414 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 560132 words processed Tue Nov 17 03:09:31 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.904751. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18418 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95301 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75772 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41927 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 600118 words processed Tue Nov 17 03:09:46 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.902280. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26679 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78784 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42895 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 640090 words processed Tue Nov 17 03:10:01 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.899970. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32106 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98686 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80528 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43563 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 680075 words processed Tue Nov 17 03:10:16 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.898482. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23656 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96492 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77655 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42490 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 720043 words processed Tue Nov 17 03:10:31 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.897260. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19364 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95445 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76079 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42032 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 760012 words processed Tue Nov 17 03:10:46 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.895295. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32286 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98389 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80625 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43721 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 800113 words processed Tue Nov 17 03:11:01 2015. [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.894884. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33351 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99734 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82273 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44557 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 400089 words processed Mon Nov 16 22:15:35 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.891812. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38013 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01644 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82973 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45382 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 440067 words processed Mon Nov 16 22:15:50 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.892423. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33348 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99718 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82221 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44417 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 480051 words processed Mon Nov 16 22:16:05 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.892530. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27665 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98281 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80641 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43546 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 520140 words processed Mon Nov 16 22:16:20 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.891831. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29448 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99765 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80550 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44067 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 560132 words processed Mon Nov 16 22:16:35 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.891016. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33687 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99367 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84515 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43717 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 600118 words processed Mon Nov 16 22:16:50 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.888579. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29289 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98187 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82346 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43299 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 640090 words processed Mon Nov 16 22:17:05 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.886273. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35123 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99584 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85155 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43911 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 680075 words processed Mon Nov 16 22:17:20 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.884796. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33479 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99230 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84343 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43837 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 720043 words processed Mon Nov 16 22:17:35 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.883469. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38145 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00395 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85538 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44442 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 760012 words processed Mon Nov 16 22:17:50 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.881407. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34510 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00165 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82612 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44485 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 800113 words processed Mon Nov 16 22:18:05 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.881039. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32317 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99593 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82885 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44008 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 840089 words processed Mon Nov 16 22:18:20 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.879681. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34847 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.00002 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83212 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44547 clock time -[SCHEDULER ITER31 LR0.00078364164096]: 880052 words processed Mon Nov 16 22:18:35 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.879010. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33226 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.99649 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82293 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44610 clock time + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34714 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99562 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81345 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44129 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 840089 words processed Tue Nov 17 03:11:16 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.893633. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20541 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95801 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76611 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42088 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 880052 words processed Tue Nov 17 03:11:31 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-1.893140. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20093 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95593 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76494 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42089 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER31 LR0.00078364164096]: Displaying result: -[SCHEDULER ITER31 LR0.00078364164096]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.153548020008> <PPL_OOV 75.6753877988> <LOGP -1746655.5789522> +[SCHEDULER ITER31 LR0.00078364164096]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 85.978253648792> <PPL_OOV 78.177745338356> <LOGP -1759789.2640874> [SCHEDULER ITER31 LR0.00078364164096]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER31 LR0.00078364164096]: shuffling training file ===PEEK ON TEST 31=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER31 LR0.00078364164096]: 40087 words processed Mon Nov 16 22:18:47 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-2.104630. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.81431 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85736 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 40087 words processed Tue Nov 17 03:11:42 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-2.099083. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69897 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77050 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER31 LR0.00078364164096]: Displaying result: -[SCHEDULER ITER31 LR0.00078364164096]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.19240725317> <PPL_OOV 127.77935275787> <LOGP -173635.55417925> +[SCHEDULER ITER31 LR0.00078364164096]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.54994153797> <PPL_OOV 126.34832321908> <LOGP -173232.37246141> [SCHEDULER ITER31 LR0.00078364164096]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 31=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER31 LR0.00078364164096]: 40095 words processed Mon Nov 16 22:19:01 2015. - [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-2.177652. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.83611 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87156 clock time +[SCHEDULER ITER31 LR0.00078364164096]: 40095 words processed Tue Nov 17 03:11:55 2015. + [SCHEDULER ITER31 LR0.00078364164096]: log prob per sample :-2.173237. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69569 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77012 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER31 LR0.00078364164096]: Displaying result: -[SCHEDULER ITER31 LR0.00078364164096]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.46985212575> <PPL_OOV 133.88550747495> <LOGP -156867.86805413> +[SCHEDULER ITER31 LR0.00078364164096]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 148.78676187249> <PPL_OOV 133.24452546283> <LOGP -156714.13786095> [SCHEDULER ITER31 LR0.00078364164096]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER31 LR0.00078364164096]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.31... @@ -6244,27 +6736,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46027284 +reading chunk 1 from 46034229 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46032134 +reading chunk 2 from 46039078 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92320569 +reading chunk 3 from 92294808 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92436778 +reading chunk 4 from 92411056 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94326910 +reading chunk 5 from 94304087 [SCHEDULER ITER32 LR0.000470184984576]: preparing parameters end. [SCHEDULER ITER32 LR0.000470184984576]: preparing layers... -(22:19:12 2015-11-16)[nerv] info: create layer: recurrentL1 -(22:19:12 2015-11-16)[nerv] info: create layer: sigmoidL1 -(22:19:12 2015-11-16)[nerv] info: create layer: combinerL1 -(22:19:12 2015-11-16)[nerv] info: create layer: outputL -(22:19:12 2015-11-16)[nerv] info: create layer: softmaxL -(22:19:12 2015-11-16)[nerv] info: create layer: selectL1 +(03:12:06 2015-11-17)[nerv] info: create layer: recurrentL1 +(03:12:06 2015-11-17)[nerv] info: create layer: sigmoidL1 +(03:12:06 2015-11-17)[nerv] info: create layer: combinerL1 +(03:12:06 2015-11-17)[nerv] info: create layer: outputL +(03:12:06 2015-11-17)[nerv] info: create layer: softmaxL +(03:12:06 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER32 LR0.000470184984576]: preparing layers end. [SCHEDULER ITER32 LR0.000470184984576]: Generate and initing TNN ... <input> selectL1 0 @@ -6292,164 +6784,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 32 LR 0.000470=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER32 LR0.000470184984576]: 40092 words processed Mon Nov 16 22:19:26 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.916048. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26171 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95883 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80566 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44503 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 80099 words processed Mon Nov 16 22:19:41 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.912091. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25652 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94291 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82405 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43543 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 120004 words processed Mon Nov 16 22:19:56 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.906112. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23081 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81599 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43010 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 160114 words processed Mon Nov 16 22:20:11 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.907220. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34214 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96863 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85944 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44583 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 200066 words processed Mon Nov 16 22:20:25 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.906412. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16548 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92498 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77632 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42840 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 240045 words processed Mon Nov 16 22:20:40 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.902609. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23239 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93688 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81926 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43210 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 280057 words processed Mon Nov 16 22:20:55 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.898541. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97568 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86179 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44741 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 320106 words processed Mon Nov 16 22:21:10 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.896520. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39319 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98049 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87338 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45371 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 360024 words processed Mon Nov 16 22:21:25 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.894722. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29980 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95150 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84758 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43743 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 400089 words processed Mon Nov 16 22:21:40 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.891648. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39247 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97758 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.88607 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45068 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 440067 words processed Mon Nov 16 22:21:55 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.892260. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38344 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96975 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.88268 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44904 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 480051 words processed Mon Nov 16 22:22:10 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.892369. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33034 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95767 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86209 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44241 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 520140 words processed Mon Nov 16 22:22:25 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.891668. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36735 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97720 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86818 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45053 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 560132 words processed Mon Nov 16 22:22:40 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.890854. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35055 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96264 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87301 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44420 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 600118 words processed Mon Nov 16 22:22:55 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.888417. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23806 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94077 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82419 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43074 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 640090 words processed Mon Nov 16 22:23:10 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.886112. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31034 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95704 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84121 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44555 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 680075 words processed Mon Nov 16 22:23:25 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.884632. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24948 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94052 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81975 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43799 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 720043 words processed Mon Nov 16 22:23:39 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.883305. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17391 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.92658 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78791 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43073 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 760012 words processed Mon Nov 16 22:23:54 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.881243. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27463 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94608 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83725 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43739 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 800113 words processed Mon Nov 16 22:24:09 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.880875. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30047 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96044 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84378 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44122 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 840089 words processed Mon Nov 16 22:24:24 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.879517. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23193 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93924 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81849 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43294 clock time -[SCHEDULER ITER32 LR0.000470184984576]: 880052 words processed Mon Nov 16 22:24:39 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.878846. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36561 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96971 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.87415 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44640 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 40092 words processed Tue Nov 17 03:12:20 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.925905. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25105 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99189 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74215 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43045 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 80099 words processed Tue Nov 17 03:12:35 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.922641. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24390 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97961 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75046 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42540 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 120004 words processed Tue Nov 17 03:12:50 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.918273. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33024 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99969 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78011 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43082 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 160114 words processed Tue Nov 17 03:13:05 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.919425. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23654 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98608 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74337 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42829 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 200066 words processed Tue Nov 17 03:13:20 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.919050. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17968 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96798 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72507 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42001 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 240045 words processed Tue Nov 17 03:13:35 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.914876. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19593 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97127 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73419 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42057 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 280057 words processed Tue Nov 17 03:13:50 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.910955. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35060 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.01331 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78344 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44102 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 320106 words processed Tue Nov 17 03:14:05 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.909159. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33350 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00778 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78131 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44078 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 360024 words processed Tue Nov 17 03:14:20 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.907436. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32096 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00184 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78172 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43448 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 400089 words processed Tue Nov 17 03:14:35 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.904438. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37396 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.01920 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79501 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44619 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 440067 words processed Tue Nov 17 03:14:50 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.905040. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28772 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99301 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77040 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43300 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 480051 words processed Tue Nov 17 03:15:05 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.905552. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18220 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96868 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73116 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42001 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 520140 words processed Tue Nov 17 03:15:20 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.904992. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23040 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98632 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74448 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42911 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 560132 words processed Tue Nov 17 03:15:35 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.904601. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21950 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97800 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74262 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42532 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 600118 words processed Tue Nov 17 03:15:50 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.902128. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18168 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96839 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73073 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41976 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 640090 words processed Tue Nov 17 03:16:05 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.899818. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21241 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97814 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74027 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42454 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 680075 words processed Tue Nov 17 03:16:20 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.898330. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97167 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73208 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42248 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 720043 words processed Tue Nov 17 03:16:35 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.897108. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32501 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00258 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78132 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43935 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 760012 words processed Tue Nov 17 03:16:50 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.895144. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23747 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98391 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74635 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42665 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 800113 words processed Tue Nov 17 03:17:05 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.894732. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24904 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99451 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74582 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43150 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 840089 words processed Tue Nov 17 03:17:20 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.893481. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18129 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96950 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.72902 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42136 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 880052 words processed Tue Nov 17 03:17:35 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-1.892987. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26471 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98988 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75875 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42916 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER32 LR0.000470184984576]: Displaying result: -[SCHEDULER ITER32 LR0.000470184984576]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.120575096429> <PPL_OOV 75.646683025421> <LOGP -1746502.4147988> +[SCHEDULER ITER32 LR0.000470184984576]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 85.945356652249> <PPL_OOV 78.15021167178> <LOGP -1759647.0532531> [SCHEDULER ITER32 LR0.000470184984576]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER32 LR0.000470184984576]: shuffling training file ===PEEK ON TEST 32=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER32 LR0.000470184984576]: 40087 words processed Mon Nov 16 22:24:51 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-2.104462. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.86504 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.89189 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 40087 words processed Tue Nov 17 03:17:46 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-2.098981. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68371 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74741 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER32 LR0.000470184984576]: Displaying result: -[SCHEDULER ITER32 LR0.000470184984576]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.13419513463> <PPL_OOV 127.72879043581> <LOGP -173621.38574441> +[SCHEDULER ITER32 LR0.000470184984576]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.54997292402> <PPL_OOV 126.31489358447> <LOGP -173222.89942476> [SCHEDULER ITER32 LR0.000470184984576]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 32=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER32 LR0.000470184984576]: 40095 words processed Mon Nov 16 22:25:05 2015. - [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-2.177560. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.81832 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86510 clock time +[SCHEDULER ITER32 LR0.000470184984576]: 40095 words processed Tue Nov 17 03:17:59 2015. + [SCHEDULER ITER32 LR0.000470184984576]: log prob per sample :-2.173132. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.68051 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75019 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER32 LR0.000470184984576]: Displaying result: -[SCHEDULER ITER32 LR0.000470184984576]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.43029704938> <PPL_OOV 133.84950308333> <LOGP -156859.25245411> +[SCHEDULER ITER32 LR0.000470184984576]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 148.78200743995> <PPL_OOV 133.21157703127> <LOGP -156706.2156885> [SCHEDULER ITER32 LR0.000470184984576]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER32 LR0.000470184984576]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.32... @@ -6458,27 +6950,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1890129 +reading chunk 1 from 1893031 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1894979 +reading chunk 2 from 1897880 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48183403 +reading chunk 3 from 48153616 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48299612 +reading chunk 4 from 48269864 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94326883 +reading chunk 5 from 94304092 [SCHEDULER ITER33 LR0.0002821109907456]: preparing parameters end. [SCHEDULER ITER33 LR0.0002821109907456]: preparing layers... -(22:25:17 2015-11-16)[nerv] info: create layer: recurrentL1 -(22:25:17 2015-11-16)[nerv] info: create layer: sigmoidL1 -(22:25:17 2015-11-16)[nerv] info: create layer: combinerL1 -(22:25:17 2015-11-16)[nerv] info: create layer: outputL -(22:25:17 2015-11-16)[nerv] info: create layer: softmaxL -(22:25:17 2015-11-16)[nerv] info: create layer: selectL1 +(03:18:10 2015-11-17)[nerv] info: create layer: recurrentL1 +(03:18:10 2015-11-17)[nerv] info: create layer: sigmoidL1 +(03:18:10 2015-11-17)[nerv] info: create layer: combinerL1 +(03:18:10 2015-11-17)[nerv] info: create layer: outputL +(03:18:10 2015-11-17)[nerv] info: create layer: softmaxL +(03:18:10 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER33 LR0.0002821109907456]: preparing layers end. [SCHEDULER ITER33 LR0.0002821109907456]: Generate and initing TNN ... <input> selectL1 0 @@ -6506,164 +6998,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 33 LR 0.000282=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER33 LR0.0002821109907456]: 40092 words processed Mon Nov 16 22:25:32 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.915948. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37873 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05257 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80758 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44690 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 80099 words processed Mon Nov 16 22:25:47 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.911974. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30407 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02275 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79752 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42885 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 120004 words processed Mon Nov 16 22:26:02 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.905997. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36776 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03898 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81471 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44061 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 160114 words processed Mon Nov 16 22:26:17 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.907108. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38599 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05030 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82585 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43966 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 200066 words processed Mon Nov 16 22:26:32 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.906303. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38039 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04046 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82876 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43840 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 240045 words processed Mon Nov 16 22:26:47 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.902497. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30417 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02875 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78821 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43547 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 280057 words processed Mon Nov 16 22:27:02 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.898435. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32106 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03941 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78453 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43937 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 320106 words processed Mon Nov 16 22:27:17 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.896415. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.41842 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06297 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82368 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45179 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 360024 words processed Mon Nov 16 22:27:32 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.894619. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30476 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02500 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79460 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43229 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 400089 words processed Mon Nov 16 22:27:47 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.891544. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32098 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03828 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79381 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43872 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 440067 words processed Mon Nov 16 22:28:02 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.892156. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33244 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03205 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80692 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43591 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 480051 words processed Mon Nov 16 22:28:17 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.892267. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30932 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03028 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79401 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43615 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 520140 words processed Mon Nov 16 22:28:32 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.891565. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32332 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03982 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79328 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43977 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 560132 words processed Mon Nov 16 22:28:47 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.890751. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.37965 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04381 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82266 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44419 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 600118 words processed Mon Nov 16 22:29:02 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.888314. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27468 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02018 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78183 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43007 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 640090 words processed Mon Nov 16 22:29:17 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.886009. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28080 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02242 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78578 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42957 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 680075 words processed Mon Nov 16 22:29:32 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.884528. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38817 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04515 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83239 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43952 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 720043 words processed Mon Nov 16 22:29:47 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.883201. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31120 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02977 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79603 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43441 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 760012 words processed Mon Nov 16 22:30:02 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.881140. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.40845 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04757 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85140 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44013 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 800113 words processed Mon Nov 16 22:30:17 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.880772. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.43033 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06615 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83252 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45343 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 840089 words processed Mon Nov 16 22:30:32 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.879414. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38990 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04882 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81922 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44482 clock time -[SCHEDULER ITER33 LR0.0002821109907456]: 880052 words processed Mon Nov 16 22:30:47 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.878742. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33591 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03636 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79804 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43945 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 40092 words processed Tue Nov 17 03:18:24 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.925795. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30445 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98948 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78349 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44117 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 80099 words processed Tue Nov 17 03:18:39 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.922541. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30629 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98132 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79152 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43297 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 120004 words processed Tue Nov 17 03:18:54 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.918166. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96313 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75505 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42270 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 160114 words processed Tue Nov 17 03:19:09 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.919317. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25463 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97668 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76904 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43033 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 200066 words processed Tue Nov 17 03:19:24 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.918948. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24645 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96929 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76920 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42704 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 240045 words processed Tue Nov 17 03:19:39 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.914772. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25413 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97001 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77205 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42914 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 280057 words processed Tue Nov 17 03:19:54 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.910853. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33016 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99397 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79425 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43606 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 320106 words processed Tue Nov 17 03:20:09 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.909060. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26905 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98127 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77300 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43138 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 360024 words processed Tue Nov 17 03:20:24 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.907339. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20421 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96047 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75594 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42142 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 400089 words processed Tue Nov 17 03:20:39 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.904342. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23568 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97431 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76346 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42801 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 440067 words processed Tue Nov 17 03:20:54 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.904943. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23638 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96967 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76591 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42458 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 480051 words processed Tue Nov 17 03:21:09 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.905458. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34836 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99198 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81707 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43935 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 520140 words processed Tue Nov 17 03:21:24 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.904896. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38216 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00775 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82165 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44493 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 560132 words processed Tue Nov 17 03:21:39 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.904507. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33979 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99251 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80893 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43719 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 600118 words processed Tue Nov 17 03:21:54 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.902034. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33918 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99123 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80622 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43653 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 640090 words processed Tue Nov 17 03:22:09 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.899723. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35581 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99231 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81622 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43753 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 680075 words processed Tue Nov 17 03:22:24 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.898234. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34082 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98978 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43762 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 720043 words processed Tue Nov 17 03:22:39 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.897013. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26919 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97643 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77939 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42934 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 760012 words processed Tue Nov 17 03:22:54 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.895049. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19291 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95694 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75181 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42085 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 800113 words processed Tue Nov 17 03:23:09 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.894638. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25030 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97432 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77447 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42919 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 840089 words processed Tue Nov 17 03:23:24 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.893386. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31438 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98677 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79788 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43495 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 880052 words processed Tue Nov 17 03:23:39 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-1.892891. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19985 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96094 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75255 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42167 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER33 LR0.0002821109907456]: Displaying result: -[SCHEDULER ITER33 LR0.0002821109907456]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.098446122277> <PPL_OOV 75.628616853786> <LOGP -1746405.9867466> +[SCHEDULER ITER33 LR0.0002821109907456]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 85.923704441301> <PPL_OOV 78.132987271132> <LOGP -1759558.0640923> [SCHEDULER ITER33 LR0.0002821109907456]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER33 LR0.0002821109907456]: shuffling training file ===PEEK ON TEST 33=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER33 LR0.0002821109907456]: 40087 words processed Mon Nov 16 22:30:59 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-2.104346. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.77699 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83693 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 40087 words processed Tue Nov 17 03:23:50 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-2.098918. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.75228 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79786 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER33 LR0.0002821109907456]: Displaying result: -[SCHEDULER ITER33 LR0.0002821109907456]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.11020664253> <PPL_OOV 127.69438835985> <LOGP -173611.74248268> +[SCHEDULER ITER33 LR0.0002821109907456]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.56542454587> <PPL_OOV 126.29195004722> <LOGP -173216.39640804> [SCHEDULER ITER33 LR0.0002821109907456]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 33=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER33 LR0.0002821109907456]: 40095 words processed Mon Nov 16 22:31:12 2015. - [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-2.177502. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69313 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77424 clock time +[SCHEDULER ITER33 LR0.0002821109907456]: 40095 words processed Tue Nov 17 03:24:03 2015. + [SCHEDULER ITER33 LR0.0002821109907456]: log prob per sample :-2.173076. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72661 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78367 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER33 LR0.0002821109907456]: Displaying result: -[SCHEDULER ITER33 LR0.0002821109907456]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.42017349093> <PPL_OOV 133.82837137992> <LOGP -156854.1947054> +[SCHEDULER ITER33 LR0.0002821109907456]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 148.79817692797> <PPL_OOV 133.19475166982> <LOGP -156702.16941644> [SCHEDULER ITER33 LR0.0002821109907456]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER33 LR0.0002821109907456]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.33... @@ -6672,27 +7164,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46027263 +reading chunk 1 from 46034229 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46032113 +reading chunk 2 from 46039078 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92320528 +reading chunk 3 from 92294817 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92436737 +reading chunk 4 from 92411065 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94326866 +reading chunk 5 from 94304098 [SCHEDULER ITER34 LR0.00016926659444736]: preparing parameters end. [SCHEDULER ITER34 LR0.00016926659444736]: preparing layers... -(22:31:23 2015-11-16)[nerv] info: create layer: recurrentL1 -(22:31:23 2015-11-16)[nerv] info: create layer: sigmoidL1 -(22:31:23 2015-11-16)[nerv] info: create layer: combinerL1 -(22:31:23 2015-11-16)[nerv] info: create layer: outputL -(22:31:23 2015-11-16)[nerv] info: create layer: softmaxL -(22:31:23 2015-11-16)[nerv] info: create layer: selectL1 +(03:24:14 2015-11-17)[nerv] info: create layer: recurrentL1 +(03:24:14 2015-11-17)[nerv] info: create layer: sigmoidL1 +(03:24:14 2015-11-17)[nerv] info: create layer: combinerL1 +(03:24:14 2015-11-17)[nerv] info: create layer: outputL +(03:24:14 2015-11-17)[nerv] info: create layer: softmaxL +(03:24:14 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER34 LR0.00016926659444736]: preparing layers end. [SCHEDULER ITER34 LR0.00016926659444736]: Generate and initing TNN ... <input> selectL1 0 @@ -6720,164 +7212,164 @@ TNN initing storage outputL -> softmaxL ===ITERATION 34 LR 0.000169=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER34 LR0.00016926659444736]: 40092 words processed Mon Nov 16 22:31:38 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.915888. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.31501 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96571 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83968 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44268 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 80099 words processed Mon Nov 16 22:31:53 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.911900. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20880 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93825 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79305 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43009 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 120004 words processed Mon Nov 16 22:32:08 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.905924. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22612 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93986 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79659 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43551 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 160114 words processed Mon Nov 16 22:32:23 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.907036. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36022 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.97928 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84043 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45583 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 200066 words processed Mon Nov 16 22:32:38 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.906236. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35291 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96520 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.86192 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44393 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 240045 words processed Mon Nov 16 22:32:53 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.902428. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25233 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94695 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80849 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43660 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 280057 words processed Mon Nov 16 22:33:08 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.898370. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25398 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95385 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80628 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44084 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 320106 words processed Mon Nov 16 22:33:23 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.896351. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24582 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95318 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80501 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43923 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 360024 words processed Mon Nov 16 22:33:38 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.894555. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28258 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95523 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82277 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44227 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 400089 words processed Mon Nov 16 22:33:53 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.891480. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36689 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.98049 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84764 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45670 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 440067 words processed Mon Nov 16 22:34:08 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.892093. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26349 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94672 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83279 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43581 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 480051 words processed Mon Nov 16 22:34:23 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.892205. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32187 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.95803 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85425 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44249 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 520140 words processed Mon Nov 16 22:34:38 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.891502. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21497 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94684 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79061 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43922 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 560132 words processed Mon Nov 16 22:34:53 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.890689. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20648 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93731 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79647 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43295 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 600118 words processed Mon Nov 16 22:35:08 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.888252. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32881 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96399 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85276 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44176 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 640090 words processed Mon Nov 16 22:35:23 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.885947. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22332 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93926 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80819 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43313 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 680075 words processed Mon Nov 16 22:35:38 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.884465. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20387 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93689 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79714 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43075 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 720043 words processed Mon Nov 16 22:35:53 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.883138. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18220 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93174 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78885 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42894 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 760012 words processed Mon Nov 16 22:36:08 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.881077. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18784 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93433 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78833 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43050 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 800113 words processed Mon Nov 16 22:36:23 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.880709. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30512 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.96310 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83496 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44920 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 840089 words processed Mon Nov 16 22:36:38 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.879351. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21311 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.94756 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79267 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42964 clock time -[SCHEDULER ITER34 LR0.00016926659444736]: 880052 words processed Mon Nov 16 22:36:53 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.878679. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18584 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:4.93586 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78086 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 40092 words processed Tue Nov 17 03:24:28 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.925728. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29765 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00794 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76573 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43528 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 80099 words processed Tue Nov 17 03:24:43 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.922479. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21433 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97511 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74177 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42221 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 120004 words processed Tue Nov 17 03:24:58 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.918100. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29228 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99511 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77056 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42880 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 160114 words processed Tue Nov 17 03:25:13 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.919249. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.39075 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.02465 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80341 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44562 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 200066 words processed Tue Nov 17 03:25:28 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.918886. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32790 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00110 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78694 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43594 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 240045 words processed Tue Nov 17 03:25:43 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.914708. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22603 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97927 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75005 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42219 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 280057 words processed Tue Nov 17 03:25:58 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.910790. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27566 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99903 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76371 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43060 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 320106 words processed Tue Nov 17 03:26:13 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.908998. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28508 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00192 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76623 clock time [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43228 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 360024 words processed Tue Nov 17 03:26:28 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.907279. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29486 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99627 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77205 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43163 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 400089 words processed Tue Nov 17 03:26:43 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.904284. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.35591 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.01662 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79469 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44023 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 440067 words processed Tue Nov 17 03:26:58 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.904885. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20308 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97616 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74087 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42064 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 480051 words processed Tue Nov 17 03:27:13 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.905401. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20790 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97760 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74440 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42115 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 520140 words processed Tue Nov 17 03:27:28 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.904838. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24948 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99370 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75398 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42939 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 560132 words processed Tue Nov 17 03:27:43 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.904450. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20194 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97528 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74448 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41982 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 600118 words processed Tue Nov 17 03:27:58 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.901977. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19817 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97518 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73966 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42003 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 640090 words processed Tue Nov 17 03:28:13 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.899665. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21155 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97756 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74788 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42138 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 680075 words processed Tue Nov 17 03:28:28 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.898176. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30675 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:5.00171 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78138 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43200 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 720043 words processed Tue Nov 17 03:28:43 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.896954. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20567 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97738 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74275 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42135 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 760012 words processed Tue Nov 17 03:28:58 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.894991. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20163 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97611 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74141 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42065 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 800113 words processed Tue Nov 17 03:29:13 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.894580. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.25497 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99583 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75600 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42980 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 840089 words processed Tue Nov 17 03:29:28 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.893328. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22151 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98173 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74726 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42305 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 880052 words processed Tue Nov 17 03:29:43 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-1.892832. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21323 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97997 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74310 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42184 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER34 LR0.00016926659444736]: Displaying result: -[SCHEDULER ITER34 LR0.00016926659444736]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.084734172296> <PPL_OOV 75.617639172754> <LOGP -1746347.382207> +[SCHEDULER ITER34 LR0.00016926659444736]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 85.910713094454> <PPL_OOV 78.122482645213> <LOGP -1759503.7827416> [SCHEDULER ITER34 LR0.00016926659444736]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER34 LR0.00016926659444736]: shuffling training file ===PEEK ON TEST 34=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER34 LR0.00016926659444736]: 40087 words processed Mon Nov 16 22:37:05 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-2.104262. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.87126 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.89181 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 40087 words processed Tue Nov 17 03:29:54 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-2.098874. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72685 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77972 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER34 LR0.00016926659444736]: Displaying result: -[SCHEDULER ITER34 LR0.00016926659444736]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.10281367216> <PPL_OOV 127.66974230171> <LOGP -173604.83233734> +[SCHEDULER ITER34 LR0.00016926659444736]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.5798651185> <PPL_OOV 126.27534311096> <LOGP -173211.68867295> [SCHEDULER ITER34 LR0.00016926659444736]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 34=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER34 LR0.00016926659444736]: 40095 words processed Mon Nov 16 22:37:19 2015. - [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-2.177462. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72227 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79652 clock time +[SCHEDULER ITER34 LR0.00016926659444736]: 40095 words processed Tue Nov 17 03:30:07 2015. + [SCHEDULER ITER34 LR0.00016926659444736]: log prob per sample :-2.173047. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.74508 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79019 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER34 LR0.00016926659444736]: Displaying result: -[SCHEDULER ITER34 LR0.00016926659444736]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.42345695287> <PPL_OOV 133.81556769051> <LOGP -156851.12982923> +[SCHEDULER ITER34 LR0.00016926659444736]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 148.81808122357> <PPL_OOV 133.18752756819> <LOGP -156700.43196078> [SCHEDULER ITER34 LR0.00016926659444736]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER34 LR0.00016926659444736]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.34... @@ -6886,27 +7378,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 1 from 1890128 +reading chunk 1 from 1893035 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 1894978 +reading chunk 2 from 1897884 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 48183383 +reading chunk 3 from 48153625 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 48299592 +reading chunk 4 from 48269872 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 5 from 94326857 +reading chunk 5 from 94304098 [SCHEDULER ITER35 LR0.00010155995666842]: preparing parameters end. [SCHEDULER ITER35 LR0.00010155995666842]: preparing layers... -(22:37:31 2015-11-16)[nerv] info: create layer: recurrentL1 -(22:37:31 2015-11-16)[nerv] info: create layer: sigmoidL1 -(22:37:31 2015-11-16)[nerv] info: create layer: combinerL1 -(22:37:31 2015-11-16)[nerv] info: create layer: outputL -(22:37:31 2015-11-16)[nerv] info: create layer: softmaxL -(22:37:31 2015-11-16)[nerv] info: create layer: selectL1 +(03:30:18 2015-11-17)[nerv] info: create layer: recurrentL1 +(03:30:18 2015-11-17)[nerv] info: create layer: sigmoidL1 +(03:30:18 2015-11-17)[nerv] info: create layer: combinerL1 +(03:30:18 2015-11-17)[nerv] info: create layer: outputL +(03:30:18 2015-11-17)[nerv] info: create layer: softmaxL +(03:30:18 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER ITER35 LR0.00010155995666842]: preparing layers end. [SCHEDULER ITER35 LR0.00010155995666842]: Generate and initing TNN ... <input> selectL1 0 @@ -6934,201 +7426,204 @@ TNN initing storage outputL -> softmaxL ===ITERATION 35 LR 0.000102=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf... batch_size: 10 chunk_size 15 -[SCHEDULER ITER35 LR0.00010155995666842]: 40092 words processed Mon Nov 16 22:37:45 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.915850. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33207 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04389 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78303 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44128 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 80099 words processed Mon Nov 16 22:38:00 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.911855. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33355 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02912 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80455 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43041 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 120004 words processed Mon Nov 16 22:38:15 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.905880. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30232 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02599 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78229 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43275 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 160114 words processed Mon Nov 16 22:38:30 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.906992. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38664 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05347 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80821 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44600 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 200066 words processed Mon Nov 16 22:38:45 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.906196. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.38234 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04419 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.81367 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44010 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 240045 words processed Mon Nov 16 22:39:00 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.902388. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26780 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02234 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76412 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43073 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 280057 words processed Mon Nov 16 22:39:15 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.898334. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36686 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04969 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79937 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44360 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 320106 words processed Mon Nov 16 22:39:30 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.896314. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.42629 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06315 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82181 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.45150 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 360024 words processed Mon Nov 16 22:39:45 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.894518. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34636 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03667 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80494 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43541 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 400089 words processed Mon Nov 16 22:40:00 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.891443. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.44342 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06249 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84518 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44768 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 440067 words processed Mon Nov 16 22:40:15 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.892056. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29859 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02699 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78824 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43088 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 480051 words processed Mon Nov 16 22:40:30 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.892169. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24346 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.01535 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76011 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42743 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 520140 words processed Mon Nov 16 22:40:45 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.891466. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.42121 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.05888 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.83472 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44584 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 560132 words processed Mon Nov 16 22:41:00 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.890653. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30812 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03046 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78805 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43131 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 600118 words processed Mon Nov 16 22:41:15 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.888215. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26956 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02105 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77205 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42864 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 640090 words processed Mon Nov 16 22:41:30 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.885910. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26360 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02018 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76931 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42902 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 680075 words processed Mon Nov 16 22:41:45 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.884429. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.36289 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04015 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.82012 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43538 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 720043 words processed Mon Nov 16 22:42:00 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.883100. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32433 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03437 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79409 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43365 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 760012 words processed Mon Nov 16 22:42:15 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.881041. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27074 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.02328 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77017 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42960 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 800113 words processed Mon Nov 16 22:42:30 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.880673. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.34223 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.04314 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79919 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43864 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 840089 words processed Mon Nov 16 22:42:45 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.879315. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.33968 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.03254 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80739 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43238 clock time -[SCHEDULER ITER35 LR0.00010155995666842]: 880052 words processed Mon Nov 16 22:43:00 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.878643. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.46555 clock time - [global_conf.timer]: time spent on tnn_actual_layer_update:5.06507 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.85735 clock time - [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44679 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 40092 words processed Tue Nov 17 03:30:32 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.925692. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27898 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98540 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77331 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43699 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 80099 words processed Tue Nov 17 03:30:47 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.922444. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.19043 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95141 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75191 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41917 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 120004 words processed Tue Nov 17 03:31:02 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.918060. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.20057 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95434 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75173 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42009 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 160114 words processed Tue Nov 17 03:31:17 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.919209. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29842 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.98329 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78759 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43558 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 200066 words processed Tue Nov 17 03:31:32 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.918851. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.27329 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97160 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78309 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42987 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 240045 words processed Tue Nov 17 03:31:47 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.914671. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.17401 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95304 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74110 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41910 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 280057 words processed Tue Nov 17 03:32:02 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.910755. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26304 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97800 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76763 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43421 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 320106 words processed Tue Nov 17 03:32:17 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.908961. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.32863 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.99574 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79290 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.44479 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 360024 words processed Tue Nov 17 03:32:31 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.907243. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16787 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95196 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73740 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42062 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 400089 words processed Tue Nov 17 03:32:46 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.904250. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.26884 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97618 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77688 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43346 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 440067 words processed Tue Nov 17 03:33:01 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.904851. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.28084 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97237 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78624 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43239 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 480051 words processed Tue Nov 17 03:33:16 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.905368. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.18262 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95268 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74910 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42127 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 520140 words processed Tue Nov 17 03:33:31 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.904805. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22768 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97042 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75949 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42944 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 560132 words processed Tue Nov 17 03:33:46 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.904417. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.29033 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97835 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78990 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43552 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 600118 words processed Tue Nov 17 03:34:01 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.901944. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.22324 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.95940 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76809 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42500 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 640090 words processed Tue Nov 17 03:34:16 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.899632. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16466 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74184 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41945 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 680075 words processed Tue Nov 17 03:34:30 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.898142. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.15424 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94933 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.73495 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.41808 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 720043 words processed Tue Nov 17 03:34:45 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.896920. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.23925 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96882 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76592 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42667 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 760012 words processed Tue Nov 17 03:35:00 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.894958. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.16796 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.94977 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.74245 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42090 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 800113 words processed Tue Nov 17 03:35:15 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.894547. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.21894 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96714 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.75815 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42772 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 840089 words processed Tue Nov 17 03:35:30 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.893295. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.24840 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.96840 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77425 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.42903 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 880052 words processed Tue Nov 17 03:35:45 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-1.892798. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:13.30958 clock time + [global_conf.timer]: time spent on tnn_actual_layer_update:4.97893 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.79501 clock time + [global_conf.timer]: time spent on tnn_actual_layer_backpropagate:2.43318 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER35 LR0.00010155995666842]: Displaying result: -[SCHEDULER ITER35 LR0.00010155995666842]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 83.07766142557> <PPL_OOV 75.611283415282> <LOGP -1746313.4479971> +[SCHEDULER ITER35 LR0.00010155995666842]: LMResult status of rnn: <SEN_CN 42068> <W_CN 887521> <PPL_NET 85.904900052553> <PPL_OOV 78.116390358636> <LOGP -1759472.2982627> [SCHEDULER ITER35 LR0.00010155995666842]: Doing on /home/slhome/txh18/workspace/nerv/play/dagL_test/train_fn_shuf end. [SCHEDULER ITER35 LR0.00010155995666842]: shuffling training file ===PEEK ON TEST 35=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER35 LR0.00010155995666842]: 40087 words processed Mon Nov 16 22:43:12 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-2.104206. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.74793 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.80520 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 40087 words processed Tue Nov 17 03:35:56 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-2.098844. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.72838 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.78421 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER35 LR0.00010155995666842]: Displaying result: -[SCHEDULER ITER35 LR0.00010155995666842]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.09855796542> <PPL_OOV 127.65294525175> <LOGP -173600.12209541> +[SCHEDULER ITER35 LR0.00010155995666842]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.58757182061> <PPL_OOV 126.26373843872> <LOGP -173208.39861241> [SCHEDULER ITER35 LR0.00010155995666842]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. ===VALIDATION 35=== [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER ITER35 LR0.00010155995666842]: 40095 words processed Mon Nov 16 22:43:25 2015. - [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-2.177435. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.70810 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.77909 clock time +[SCHEDULER ITER35 LR0.00010155995666842]: 40095 words processed Tue Nov 17 03:36:09 2015. + [SCHEDULER ITER35 LR0.00010155995666842]: log prob per sample :-2.173032. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69248 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76343 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER ITER35 LR0.00010155995666842]: Displaying result: -[SCHEDULER ITER35 LR0.00010155995666842]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 149.42787770488> <PPL_OOV 133.80838513334> <LOGP -156849.41038012> +[SCHEDULER ITER35 LR0.00010155995666842]: LMResult status of rnn: <SEN_CN 3370> <W_CN 70390> <PPL_NET 148.83228164819> <PPL_OOV 133.18468116265> <LOGP -156699.74735103> [SCHEDULER ITER35 LR0.00010155995666842]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.valid.txt.adds end. [SCHEDULER ITER35 LR0.00010155995666842]: PPL improves, saving net to file /home/slhome/txh18/workspace/nerv/play/dagL_test/params.35... ===VALIDATION PPL record=== -<ITER3 LR1.00000 train:219.110 valid:237.806 test:229.721> -<ITER4 LR1.00000 train:213.497 valid:210.507 test:200.650> -<ITER5 LR1.00000 train:187.191 valid:194.771 test:185.904> -<ITER6 LR1.00000 train:172.331 valid:185.758 test:177.039> -<ITER7 LR1.00000 train:161.726 valid:180.163 test:171.158> -<ITER8 LR1.00000 train:153.561 valid:176.548 test:167.447> -<ITER9 LR1.00000 train:145.389 valid:173.264 test:164.459> -<ITER10 LR1.00000 train:139.377 valid:169.832 test:161.063> -<ITER11 LR1.00000 train:133.680 valid:167.319 test:158.997> -<ITER12 LR1.00000 train:129.283 valid:167.442 test:158.810> -<ITER13 LR1.00000 train:129.281 valid:167.222 test:158.683> -<ITER14 LR1.00000 train:125.548 valid:166.375 test:157.911> -<ITER15 LR1.00000 train:121.168 valid:163.980 test:155.683> -<ITER16 LR1.00000 train:118.240 valid:162.749 test:154.441> -<ITER17 LR1.00000 train:115.069 valid:161.691 test:153.464> -<ITER18 LR0.60000 train:100.370 valid:153.422 test:146.047> -<ITER19 LR0.36000 train:91.627 valid:147.134 test:140.123> -<ITER20 LR0.21600 train:86.075 valid:142.463 test:135.614> -<ITER21 LR0.12960 train:82.462 valid:139.427 test:132.753> -<ITER22 LR0.07776 train:80.073 valid:137.637 test:131.122> -<ITER23 LR0.04666 train:78.540 valid:136.420 test:130.086> -<ITER24 LR0.02799 train:77.515 valid:135.608 test:129.381> -<ITER25 LR0.01680 train:76.810 valid:134.993 test:128.863> -<ITER26 LR0.01008 train:76.363 valid:134.569 test:128.497> -<ITER27 LR0.00605 train:76.082 valid:134.296 test:128.246> -<ITER28 LR0.00363 train:75.904 valid:134.130 test:128.072> -<ITER29 LR0.00218 train:75.792 valid:134.022 test:127.947> -<ITER30 LR0.00131 train:75.720 valid:133.943 test:127.851> -<ITER31 LR0.00078 train:75.675 valid:133.886 test:127.779> -<ITER32 LR0.00047 train:75.647 valid:133.850 test:127.729> -<ITER33 LR0.00028 train:75.629 valid:133.828 test:127.694> -<ITER34 LR0.00017 train:75.618 valid:133.816 test:127.670> -<ITER35 LR0.00010 train:75.611 valid:133.808 test:127.653> +<ITER0 LR0.00000 train:0.000 valid:10730.852 test:0.000> +<ITER1 LR1.00000 train:472.885 valid:340.957 test:331.060> +<ITER2 LR1.00000 train:315.415 valid:263.058 test:252.204> +<ITER3 LR1.00000 train:248.017 valid:231.238 test:220.420> +<ITER4 LR1.00000 train:221.748 valid:213.352 test:203.341> +<ITER5 LR1.00000 train:239.789 valid:217.327 test:206.641> +<ITER6 LR1.00000 train:229.017 valid:208.234 test:197.452> +<ITER7 LR1.00000 train:188.288 valid:195.406 test:185.087> +<ITER8 LR1.00000 train:176.504 valid:190.556 test:180.180> +<ITER9 LR1.00000 train:163.001 valid:181.314 test:171.455> +<ITER10 LR1.00000 train:152.863 valid:176.607 test:166.659> +<ITER11 LR1.00000 train:145.302 valid:173.269 test:163.397> +<ITER12 LR1.00000 train:139.085 valid:170.261 test:160.532> +<ITER13 LR1.00000 train:133.626 valid:168.330 test:158.528> +<ITER14 LR1.00000 train:129.605 valid:167.660 test:157.826> +<ITER15 LR1.00000 train:124.883 valid:166.862 test:156.938> +<ITER16 LR1.00000 train:121.090 valid:165.636 test:155.485> +<ITER17 LR1.00000 train:117.612 valid:165.440 test:155.345> +<ITER18 LR0.60000 train:102.832 valid:153.428 test:144.382> +<ITER19 LR0.36000 train:94.069 valid:145.646 test:137.237> +<ITER20 LR0.21600 train:88.548 valid:141.111 test:133.098> +<ITER21 LR0.12960 train:84.953 valid:138.557 test:130.768> +<ITER22 LR0.07776 train:82.567 valid:136.938 test:129.378> +<ITER23 LR0.04666 train:81.038 valid:135.768 test:128.403> +<ITER24 LR0.02799 train:80.013 valid:135.010 test:127.781> +<ITER25 LR0.01680 train:79.309 valid:134.439 test:127.341> +<ITER26 LR0.01008 train:78.864 valid:134.000 test:127.006> +<ITER27 LR0.00605 train:78.583 valid:133.695 test:126.759> +<ITER28 LR0.00363 train:78.404 valid:133.501 test:126.589> +<ITER29 LR0.00218 train:78.292 valid:133.379 test:126.474> +<ITER30 LR0.00131 train:78.222 valid:133.299 test:126.399> +<ITER31 LR0.00078 train:78.178 valid:133.245 test:126.348> +<ITER32 LR0.00047 train:78.150 valid:133.212 test:126.315> +<ITER33 LR0.00028 train:78.133 valid:133.195 test:126.292> +<ITER34 LR0.00017 train:78.122 valid:133.188 test:126.275> +<ITER35 LR0.00010 train:78.116 valid:133.185 test:126.264> ===FINAL TEST=== [SCHEDULER FINAL_TEST]: preparing parameters... @@ -7136,27 +7631,27 @@ batch_size: 10 chunk_size 15 reading chunk 0 from 0 metadata: return {type="nerv.LinearTransParam",id="ltp_ih"} -reading chunk 1 from 46027263 +reading chunk 1 from 46034228 metadata: return {type="nerv.BiasParam",id="bp_h"} -reading chunk 2 from 46032113 +reading chunk 2 from 46039077 metadata: return {type="nerv.LinearTransParam",id="ltp_ho"} -reading chunk 3 from 92320518 +reading chunk 3 from 92294821 metadata: return {type="nerv.BiasParam",id="bp_o"} -reading chunk 4 from 92436727 +reading chunk 4 from 92411068 metadata: return {type="nerv.LinearTransParam",id="ltp_hh"} -reading chunk 5 from 94326857 +reading chunk 5 from 94304103 [SCHEDULER FINAL_TEST]: preparing parameters end. [SCHEDULER FINAL_TEST]: preparing layers... -(22:43:36 2015-11-16)[nerv] info: create layer: recurrentL1 -(22:43:36 2015-11-16)[nerv] info: create layer: sigmoidL1 -(22:43:36 2015-11-16)[nerv] info: create layer: combinerL1 -(22:43:36 2015-11-16)[nerv] info: create layer: outputL -(22:43:36 2015-11-16)[nerv] info: create layer: softmaxL -(22:43:36 2015-11-16)[nerv] info: create layer: selectL1 +(03:36:20 2015-11-17)[nerv] info: create layer: recurrentL1 +(03:36:20 2015-11-17)[nerv] info: create layer: sigmoidL1 +(03:36:20 2015-11-17)[nerv] info: create layer: combinerL1 +(03:36:20 2015-11-17)[nerv] info: create layer: outputL +(03:36:20 2015-11-17)[nerv] info: create layer: softmaxL +(03:36:20 2015-11-17)[nerv] info: create layer: selectL1 [SCHEDULER FINAL_TEST]: preparing layers end. [SCHEDULER FINAL_TEST]: Generate and initing TNN ... <input> selectL1 0 @@ -7183,11 +7678,11 @@ TNN initing storage outputL -> softmaxL [SCHEDULER FINAL_TEST]: Initing TNN end. [LOG]LMSeqReader: opening file /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds... batch_size: 10 chunk_size 15 -[SCHEDULER FINAL_TEST]: 40087 words processed Mon Nov 16 22:43:43 2015. - [SCHEDULER FINAL_TEST]: log prob per sample :-2.104206. - [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.78324 clock time - [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.84086 clock time +[SCHEDULER FINAL_TEST]: 40087 words processed Tue Nov 17 03:36:27 2015. + [SCHEDULER FINAL_TEST]: log prob per sample :-2.098844. + [global_conf.timer]: time spent on most_out_loop_lmprocessfile:5.69883 clock time + [global_conf.timer]: time spent on tnn_actual_layer_propagate:3.76204 clock time [LOG]LMSeqReader: file expires, closing. [SCHEDULER FINAL_TEST]: Displaying result: -[SCHEDULER FINAL_TEST]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 146.09855762739> <PPL_OOV 127.65294490737> <LOGP -173600.12199883> +[SCHEDULER FINAL_TEST]: LMResult status of rnn: <SEN_CN 3761> <W_CN 78669> <PPL_NET 144.587572525> <PPL_OOV 126.263739092> <LOGP -173208.39879763> [SCHEDULER FINAL_TEST]: Doing on /home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata/ptb.test.txt.adds end. diff --git a/nerv/examples/lmptb/rnn/tnn.lua b/nerv/examples/lmptb/rnn/tnn.lua index d10ab82..c2e397c 100644 --- a/nerv/examples/lmptb/rnn/tnn.lua +++ b/nerv/examples/lmptb/rnn/tnn.lua @@ -291,11 +291,21 @@ function TNN:getfeed_from_reader(reader) return got_new, feeds_now end -function TNN:move_right_to_nextmb() --move output history activations of 1..chunk_size to 1-chunk_size..0 - for t = 1, self.chunk_size, 1 do +function TNN:move_right_to_nextmb(list_t) --move output history activations of 1..chunk_size to 1-chunk_size..0 + if list_t == nil then + list_t = {} + for i = 1, self.chunk_size do + list_t[i] = i - self.chunk_size + end + end + for i = 1, #list_t do + t = list_t[i] + if t < 1 - self.chunk_size or t > 0 then + nerv.error("MB move range error") + end for id, ref in pairs(self.layers) do for p = 1, #ref.dim_out do - ref.outputs_m[t - self.chunk_size][p]:copy_fromd(ref.outputs_m[t][p]) + ref.outputs_m[t][p]:copy_fromd(ref.outputs_m[t + self.chunk_size][p]) end end end |