diff options
| -rw-r--r-- | nerv/examples/lmptb/m-tests/tnn_test.lua | 3 | ||||
| -rw-r--r-- | nerv/examples/lmptb/rnn/tnn.lua | 31 |
2 files changed, 20 insertions, 14 deletions
diff --git a/nerv/examples/lmptb/m-tests/tnn_test.lua b/nerv/examples/lmptb/m-tests/tnn_test.lua index ac9f570..40e332c 100644 --- a/nerv/examples/lmptb/m-tests/tnn_test.lua +++ b/nerv/examples/lmptb/m-tests/tnn_test.lua @@ -297,6 +297,7 @@ prepare_parameters(global_conf, true) --randomly generate parameters print("===INITIAL VALIDATION===") local tnn, paramRepo = load_net(global_conf) local result = lm_process_file(global_conf, global_conf.valid_fn, tnn, false) --false update! +nerv.LMUtil.wait(3) ppl_rec = {} lr_rec = {} ppl_rec[0] = result:ppl_net("rnn") ppl_last = ppl_rec[0] @@ -316,7 +317,7 @@ for iter = 1, global_conf.max_iter, 1 do ppl_rec[iter] = result:ppl_net("rnn") lr_rec[iter] = global_conf.lrate if (ppl_last / ppl_rec[iter] < 1.03 or lr_half == true) then - global_conf.lrate = (global_conf.lrate / 2) + global_conf.lrate = (global_conf.lrate * 0.6) lr_half = true end if (ppl_rec[iter] < ppl_last) then diff --git a/nerv/examples/lmptb/rnn/tnn.lua b/nerv/examples/lmptb/rnn/tnn.lua index dfcef63..10b501e 100644 --- a/nerv/examples/lmptb/rnn/tnn.lua +++ b/nerv/examples/lmptb/rnn/tnn.lua @@ -369,6 +369,7 @@ function TNN:propagate_dfs(ref, t) --ok, do propagate --print("debug ok, propagating"); + --[[ if (bit.band(self.feeds_now.flagsPack_now[t], bit.bor(nerv.TNN.FC.SEQ_START, nerv.TNN.FC.SEQ_END)) > 0) then --flush cross-border history for i = 1, self.batch_size do local seq_start = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_START) @@ -383,15 +384,16 @@ function TNN:propagate_dfs(ref, t) end end end + ]]-- ref.layer:propagate(ref.inputs_m[t], ref.outputs_m[t], t) --propagate! if (bit.band(self.feeds_now.flagsPack_now[t], bit.bor(nerv.TNN.FC.SEQ_START, nerv.TNN.FC.SEQ_END)) > 0) then --restore cross-border history for i = 1, self.batch_size do local seq_start = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_START) local seq_end = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_END) if (seq_start > 0 or seq_end > 0) then - for p, conn in pairs(ref.i_conns_p) do - if ((ref.i_conns_p[p].time > 0 and seq_start > 0) or (ref.i_conns_p[p].time < 0 and seq_end > 0)) then - ref.inputs_m[t][p][i - 1]:copy_fromd(ref.inputs_matbak_p[p][i - 1]) + for p, conn in pairs(ref.o_conns_p) do + if ((ref.o_conns_p[p].time > 0 and seq_end > 0) or (ref.o_conns_p[p].time < 0 and seq_start > 0)) then + ref.outputs_m[t][p][i - 1]:fill(self.gconf.nn_act_default) end end end @@ -485,25 +487,28 @@ function TNN:backpropagate_dfs(ref, t, do_update) --ok, do back_propagate --print("debug ok, back-propagating(or updating)") - if (bit.band(self.feeds_now.flagsPack_now[t], bit.bor(nerv.TNN.FC.SEQ_START, nerv.TNN.FC.SEQ_END)) > 0) then --flush cross-border errors + if (do_update == false) then + ref.layer:back_propagate(ref.err_inputs_m[t], ref.err_outputs_m[t], ref.inputs_m[t], ref.outputs_m[t], t) + else + --print(ref.err_inputs_m[t][1]) + ref.layer:update(ref.err_inputs_m[t], ref.inputs_m[t], ref.outputs_m[t], t) + end + + if (do_update == false and bit.band(self.feeds_now.flagsPack_now[t], bit.bor(nerv.TNN.FC.SEQ_START, nerv.TNN.FC.SEQ_END)) > 0) then --flush cross-border errors for i = 1, self.batch_size do local seq_start = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_START) local seq_end = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_END) if (seq_start > 0 or seq_end > 0) then - for p, conn in pairs(ref.o_conns_p) do - if ((ref.o_conns_p[p].time > 0 and seq_end > 0) or (ref.o_conns_p[p].time < 0 and seq_start > 0)) then --cross-border, set to zero - ref.err_inputs_m[t][p][i - 1]:fill(0) + for p, conn in pairs(ref.i_conns_p) do + if ((ref.i_conns_p[p].time > 0 and seq_start > 0) or (ref.i_conns_p[p].time < 0 and seq_end > 0)) then --cross-border, set to zero + ref.err_outputs_m[t][p][i - 1]:fill(0) end end end end end - if (do_update == false) then - ref.layer:back_propagate(ref.err_inputs_m[t], ref.err_outputs_m[t], ref.inputs_m[t], ref.outputs_m[t], t) - else - --print(ref.err_inputs_m[t][1]) - ref.layer:update(ref.err_inputs_m[t], ref.inputs_m[t], ref.outputs_m[t], t) - end + + for i = 1, #ref.dim_in do if (ref.err_outputs_b[t][i] == true) then nerv.error("this time's outputs_b should be false") |