diff options
Diffstat (limited to 'nerv/examples/lmptb')
-rw-r--r-- | nerv/examples/lmptb/grulm_ptb_main.lua | 16 | ||||
-rw-r--r-- | nerv/examples/lmptb/lm_sampler.lua | 60 | ||||
-rw-r--r-- | nerv/examples/lmptb/lmptb/layer/gru_t.lua | 114 | ||||
-rw-r--r-- | nerv/examples/lmptb/lmptb/layer/init.lua | 2 | ||||
-rw-r--r-- | nerv/examples/lmptb/lmptb/lmseqreader.lua | 34 | ||||
-rw-r--r-- | nerv/examples/lmptb/lmptb/lmutil.lua | 13 | ||||
-rw-r--r-- | nerv/examples/lmptb/lmptb/lmvocab.lua | 6 | ||||
-rw-r--r-- | nerv/examples/lmptb/lstmlm_ptb_main.lua | 6 | ||||
-rw-r--r-- | nerv/examples/lmptb/m-tests/lm_sampler_test.lua (renamed from nerv/examples/lmptb/sample_grulm_ptb_main.lua) | 167 | ||||
-rw-r--r-- | nerv/examples/lmptb/m-tests/lmseqreader_test.lua | 10 | ||||
-rw-r--r-- | nerv/examples/lmptb/m-tests/some-text | 2 | ||||
-rw-r--r-- | nerv/examples/lmptb/rnnlm_ptb_main.lua | 37 |
12 files changed, 235 insertions, 232 deletions
diff --git a/nerv/examples/lmptb/grulm_ptb_main.lua b/nerv/examples/lmptb/grulm_ptb_main.lua index ef5d7f9..4a3f39f 100644 --- a/nerv/examples/lmptb/grulm_ptb_main.lua +++ b/nerv/examples/lmptb/grulm_ptb_main.lua @@ -198,6 +198,7 @@ qdata_dir = root_dir .. '/ptb/questionGen/gen' global_conf = { lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, cumat_type = nerv.CuMatrixFloat, + select_gpu = 0, mmat_type = nerv.MMatrixFloat, nn_act_default = 0, @@ -259,7 +260,7 @@ global_conf = { elseif (set == "twitter") then data_dir = root_dir .. '/twitter_new/DATA' -train_fn = data_dir .. '/twitter.choose2.adds' +train_fn = data_dir .. '/twitter.choose.adds' valid_fn = data_dir .. '/twitter.valid.adds' test_fn = data_dir .. '/comm.test.choose-ppl.adds' vocab_fn = data_dir .. '/twitter.choose.train.vocab' @@ -359,7 +360,14 @@ commands = nerv.SUtil.parse_commands_set(commands_str) if start_lr ~= nil then global_conf.lrate = start_lr end - + +nerv.printf("detecting gconf.select_gpu...\n") +if global_conf.select_gpu then + nerv.printf("select gpu to %d\n", global_conf.select_gpu) + global_conf.cumat_type.select_gpu(global_conf.select_gpu) + nerv.LMUtil.wait(1) +end + nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir) nerv.LMUtil.wait(2) os.execute("mkdir -p "..global_conf.work_dir) @@ -388,10 +396,10 @@ nerv.LMUtil.wait(2) math.randomseed(1) -local vocab = nerv.LMVocab() +local vocab = nerv.LMVocab(global_conf) global_conf["vocab"] = vocab nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) -global_conf.vocab:build_file(global_conf.vocab_fn, false) +global_conf.vocab:build_file(global_conf.vocab_fn) ppl_rec = {} local final_iter = -1 diff --git a/nerv/examples/lmptb/lm_sampler.lua b/nerv/examples/lmptb/lm_sampler.lua index c25a75c..c9adf85 100644 --- a/nerv/examples/lmptb/lm_sampler.lua +++ b/nerv/examples/lmptb/lm_sampler.lua @@ -3,31 +3,34 @@ local LMSampler = nerv.class('nerv.LMSampler') function LMSampler:__init(global_conf) self.log_pre = "LMSampler" self.gconf = global_conf + self.batch_size = self.gconf.batch_size + self.chunk_size = self.gconf.chunk_size --largest sample sentence length self.vocab = self.gconf.vocab self.sen_end_token = self.vocab.sen_end_token self.sen_end_id = self.vocab:get_word_str(self.sen_end_token).id + + self.loaded = false end -function LMSampler:load_dagL(dagL) - self.batch_size = self.gconf.batch_size - self.chunk_size = self.gconf.chunk_size - +function LMSampler:load_dagL(dagL) nerv.printf("%s loading dagL\n", self.log_pre) self.dagL = dagL + self.dagL:init(self.batch_size) self.dagL_inputs = {} - self.dagL_inputs[1] = global_conf.cumat_type(global_conf.batch_size, 1) + self.dagL_inputs[1] = self.gconf.cumat_type(self.gconf.batch_size, 1) self.dagL_inputs[1]:fill(self.sen_end_id - 1) - self.dagL_inputs[2] = global_conf.cumat_type(global_conf.batch_size, global_conf.hidden_size) + self.dagL_inputs[2] = self.gconf.cumat_type(self.gconf.batch_size, self.gconf.hidden_size) self.dagL_inputs[2]:fill(0) self.dagL_outputs = {} - self.dagL_outputs[1] = global_conf.cumat_type(global_conf.batch_size, global_conf.vocab:size()) - self.dagL_outputs[2] = global_conf.cumat_type(global_conf.batch_size, global_conf.hidden_size) + self.dagL_outputs[1] = self.gconf.cumat_type(self.gconf.batch_size, self.gconf.vocab:size()) + self.dagL_outputs[2] = self.gconf.cumat_type(self.gconf.batch_size, self.gconf.hidden_size) - self.smout_d = global_conf.cumat_type(self.batch_size, self.vocab:size()) - self.smout_h = global_conf.mmat_type(self.batch_size, self.vocab:size()) + self.smout_d = self.gconf.cumat_type(self.batch_size, self.vocab:size()) + self.ssout_d = self.gconf.cumat_type(self.batch_size, self.vocab:size()) + self.ssout_h = self.gconf.mmat_type(self.batch_size, self.vocab:size()) self.store = {} for i = 1, self.batch_size do @@ -38,11 +41,31 @@ function LMSampler:load_dagL(dagL) self.store[i][1].p = 0 end self.repo = {} + + self.loaded = true end -function LMSampler:sample_to_store(smout) +function LMSampler:sample_to_store(ssout) --private for i = 1, self.batch_size do local ran = math.random() + local id = 1 + local low = 0 + local high = ssout:ncol() - 1 + if ssout[i - 1][high] < 0.9999 or ssout[i - 1][high] > 1.0001 then + nerv.error("%s ERROR, softmax output summation(%f) seems to have some problem", self.log_pre, ssout[i - 1][high]) + end + if ssout[i - 1][low] < ran then + while low + 1 < high do + local mid = math.floor((low + high) / 2) + if ssout[i - 1][mid] < ran then + low = mid + else + high = mid + end + end + id = high + 1 + end + --[[ local s = 0 local id = self.vocab:size() for j = 0, self.vocab:size() - 1 do @@ -52,19 +75,25 @@ function LMSampler:sample_to_store(smout) break end end + ]]-- if #self.store[i] >= self.chunk_size - 2 then id = self.sen_end_id end local tmp = {} tmp.w = self.vocab:get_word_id(id).str tmp.id = id - tmp.p = smout[i - 1][id - 1] + if id == 1 then + tmp.p = ssout[i - 1][id - 1] + else + tmp.p = ssout[i - 1][id - 1] - ssout[i - 1][id - 2] + end table.insert(self.store[i], tmp) end end ---Returns: LMResult function LMSampler:lm_sample_rnn_dagL(sample_num, p_conf) + assert(self.loaded == true) + local dagL = self.dagL local inputs = self.dagL_inputs local outputs = self.dagL_outputs @@ -74,9 +103,10 @@ function LMSampler:lm_sample_rnn_dagL(sample_num, p_conf) inputs[2]:copy_fromd(outputs[2]) --copy hidden activation self.smout_d:softmax(outputs[1]) - self.smout_d:copy_toh(self.smout_h) + self.ssout_d:prefixsum_row(self.smout_d) + self.ssout_d:copy_toh(self.ssout_h) - self:sample_to_store(self.smout_h) + self:sample_to_store(self.ssout_h) for i = 1, self.batch_size do inputs[1][i - 1][0] = self.store[i][#self.store[i]].id - 1 if self.store[i][#self.store[i]].id == self.sen_end_id then --meet a sentence end diff --git a/nerv/examples/lmptb/lmptb/layer/gru_t.lua b/nerv/examples/lmptb/lmptb/layer/gru_t.lua deleted file mode 100644 index 8f15cc8..0000000 --- a/nerv/examples/lmptb/lmptb/layer/gru_t.lua +++ /dev/null @@ -1,114 +0,0 @@ -local GRULayerT = nerv.class('nerv.GRULayerT', 'nerv.LayerT') - -function GRULayerT:__init(id, global_conf, layer_conf) - --input1:x input2:h input3:c(h^~) - self.id = id - self.dim_in = layer_conf.dim_in - self.dim_out = layer_conf.dim_out - self.gconf = global_conf - - if self.dim_in[2] ~= self.dim_out[1] then - nerv.error("dim_in[2](%d) mismatch with dim_out[1](%d)", self.dim_in[2], self.dim_out[1]) - end - - --prepare a DAGLayerT to hold the lstm structure - local pr = layer_conf.pr - if pr == nil then - pr = nerv.ParamRepo() - end - - local function ap(str) - return self.id .. '.' .. str - end - - local layers = { - ["nerv.CombinerLayer"] = { - [ap("inputXDup")] = {{}, {["dim_in"] = {self.dim_in[1]}, - ["dim_out"] = {self.dim_in[1], self.dim_in[1], self.dim_in[1]}, ["lambda"] = {1}}}, - [ap("inputHDup")] = {{}, {["dim_in"] = {self.dim_in[2]}, - ["dim_out"] = {self.dim_in[2], self.dim_in[2], self.dim_in[2], self.dim_in[2], self.dim_in[2]}, ["lambda"] = {1}}}, - [ap("updateGDup")] = {{}, {["dim_in"] = {self.dim_in[2]}, - ["dim_out"] = {self.dim_in[2], self.dim_in[2]}, ["lambda"] = {1}}}, - [ap("updateMergeL")] = {{}, {["dim_in"] = {self.dim_in[2], self.dim_in[2], self.dim_in[2]}, ["dim_out"] = {self.dim_out[1]}, - ["lambda"] = {1, -1, 1}}}, - }, - ["nerv.AffineLayer"] = { - [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, ["dim_out"] = {self.dim_out[1]}, ["pr"] = pr}}, - }, - ["nerv.TanhLayer"] = { - [ap("mainTanhL")] = {{}, {["dim_in"] = {self.dim_out[1]}, ["dim_out"] = {self.dim_out[1]}}}, - }, - ["nerv.GateFLayer"] = { - [ap("resetGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, - ["dim_out"] = {self.dim_in[2]}, ["pr"] = pr}}, - [ap("updateGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]}, - ["dim_out"] = {self.dim_in[2]}, ["pr"] = pr}}, - }, - ["nerv.ElemMulLayer"] = { - [ap("resetGMulL")] = {{}, {["dim_in"] = {self.dim_in[2], self.dim_in[2]}, ["dim_out"] = {self.dim_in[2]}}}, - [ap("updateGMulCL")] = {{}, {["dim_in"] = {self.dim_in[2], self.dim_in[2]}, ["dim_out"] = {self.dim_in[2]}}}, - [ap("updateGMulHL")] = {{}, {["dim_in"] = {self.dim_in[2], self.dim_in[2]}, ["dim_out"] = {self.dim_in[2]}}}, - }, - } - - local layerRepo = nerv.LayerRepo(layers, pr, global_conf) - - local connections_t = { - ["<input>[1]"] = ap("inputXDup[1]"), - ["<input>[2]"] = ap("inputHDup[1]"), - - [ap("inputXDup[1]")] = ap("resetGateL[1]"), - [ap("inputHDup[1]")] = ap("resetGateL[2]"), - [ap("inputXDup[2]")] = ap("updateGateL[1]"), - [ap("inputHDup[2]")] = ap("updateGateL[2]"), - [ap("updateGateL[1]")] = ap("updateGDup[1]"), - - [ap("resetGateL[1]")] = ap("resetGMulL[1]"), - [ap("inputHDup[3]")] = ap("resetGMulL[2]"), - - [ap("inputXDup[3]")] = ap("mainAffineL[1]"), - [ap("resetGMulL[1]")] = ap("mainAffineL[2]"), - [ap("mainAffineL[1]")] = ap("mainTanhL[1]"), - - [ap("updateGDup[1]")] = ap("updateGMulHL[1]"), - [ap("inputHDup[4]")] = ap("updateGMulHL[2]"), - [ap("updateGDup[2]")] = ap("updateGMulCL[1]"), - [ap("mainTanhL[1]")] = ap("updateGMulCL[2]"), - - [ap("inputHDup[5]")] = ap("updateMergeL[1]"), - [ap("updateGMulHL[1]")] = ap("updateMergeL[2]"), - [ap("updateGMulCL[1]")] = ap("updateMergeL[3]"), - - [ap("updateMergeL[1]")] = "<output>[1]", - } - - self.dagL = nerv.DAGLayerT(self.id, global_conf, - {["dim_in"] = self.dim_in, ["dim_out"] = self.dim_out, ["sub_layers"] = layerRepo, - ["connections"] = connections_t}) - - self:check_dim_len(2, 1) -- x, h and h -end - -function GRULayerT:init(batch_size, chunk_size) - self.dagL:init(batch_size, chunk_size) -end - -function GRULayerT:batch_resize(batch_size, chunk_size) - self.dagL:batch_resize(batch_size, chunk_size) -end - -function GRULayerT:update(bp_err, input, output, t) - self.dagL:update(bp_err, input, output, t) -end - -function GRULayerT:propagate(input, output, t) - self.dagL:propagate(input, output, t) -end - -function GRULayerT:back_propagate(bp_err, next_bp_err, input, output, t) - self.dagL:back_propagate(bp_err, next_bp_err, input, output, t) -end - -function GRULayerT:get_params() - return self.dagL:get_params() -end diff --git a/nerv/examples/lmptb/lmptb/layer/init.lua b/nerv/examples/lmptb/lmptb/layer/init.lua index b345244..ceae009 100644 --- a/nerv/examples/lmptb/lmptb/layer/init.lua +++ b/nerv/examples/lmptb/lmptb/layer/init.lua @@ -1,6 +1,6 @@ require 'lmptb.layer.select_linear' require 'lmptb.layer.affine_recurrent_plusvec' -require 'lmptb.layer.gru_t' +--require 'lmptb.layer.gru_t' require 'lmptb.layer.lm_affine_recurrent' diff --git a/nerv/examples/lmptb/lmptb/lmseqreader.lua b/nerv/examples/lmptb/lmptb/lmseqreader.lua index 0f29f8b..1272929 100644 --- a/nerv/examples/lmptb/lmptb/lmseqreader.lua +++ b/nerv/examples/lmptb/lmptb/lmseqreader.lua @@ -28,6 +28,10 @@ function LMReader:__init(global_conf, batch_size, chunk_size, vocab, r_conf) if r_conf.compressed_label == true then self.compressed_label = true end + self.same_io = false + if r_conf.same_io == true then --can be used to train P(wi|w1..(i-1),(i+1)..n) + self.same_io = true + end end --fn: string @@ -36,9 +40,9 @@ function LMReader:open_file(fn) if (self.fh ~= nil) then nerv.error("%s error: in open_file(fn is %s), file handle not nil.", self.log_pre, fn) end - printf("%s opening file %s...\n", self.log_pre, fn) - print(self.log_pre, "batch_size:", self.batch_size, "chunk_size", self.chunk_size) - print(self.log_pre, "se_mode:", self.se_mode) + nerv.printf("%s opening file %s...\n", self.log_pre, fn) + nerv.printf("%s batch_size:%d chunk_size:%d\n", self.log_pre, self.batch_size, self.chunk_size) + nerv.printf("%s se_mode:%s same_io:%s\n", self.log_pre, tostring(self.se_mode), tostring(self.same_io)) self.fh = io.open(fn, "r") self.streams = {} for i = 1, self.batch_size, 1 do @@ -132,12 +136,15 @@ function LMReader:get_batch(feeds) else self:refresh_stream(i) if st.store[st.head] ~= nil then - inputs_s[j][i] = st.store[st.head] - --inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head]).id - 1 - self.bak_inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head]).id - 1 + if self.same_io == false then + inputs_s[j][i] = st.store[st.head] + self.bak_inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head]).id - 1 + else + inputs_s[j][i] = st.store[st.head + 1] + self.bak_inputs_m[j][1][i - 1][0] = self.vocab:get_word_str(st.store[st.head + 1]).id - 1 + end else inputs_s[j][i] = self.vocab.null_token - --inputs_m[j][1][i - 1][0] = 0 self.bak_inputs_m[j][1][i - 1][0] = 0 end if st.store[st.head + 1] ~= nil then @@ -148,7 +155,7 @@ function LMReader:get_batch(feeds) inputs_m[j][2][i - 1][self.vocab:get_word_str(st.store[st.head + 1]).id - 1] = 1 end else - if (inputs_s[j][i] ~= self.vocab.null_token) then + if inputs_s[j][i] ~= self.vocab.null_token then nerv.error("reader error : input not null but label is null_token") end labels_s[j][i] = self.vocab.null_token @@ -159,6 +166,9 @@ function LMReader:get_batch(feeds) end flags[j][i] = bit.bor(flags[j][i], nerv.TNN.FC.SEQ_NORM) --has both input and label got_new = true + if st.store[st.head] == self.vocab.sen_end_token then + flags[j][i] = bit.bor(flags[j][i], nerv.TNN.FC.SEQ_START) + end st.store[st.head] = nil st.head = st.head + 1 if labels_s[j][i] == self.vocab.sen_end_token then @@ -169,10 +179,7 @@ function LMReader:get_batch(feeds) end_stream = true --meet sentence end, this stream ends now end end - if inputs_s[j][i] == self.vocab.sen_end_token then - flags[j][i] = bit.bor(flags[j][i], nerv.TNN.FC.SEQ_START) - end - end + end end end end @@ -190,7 +197,7 @@ function LMReader:get_batch(feeds) --check for self.al_sen_start for i = 1, self.batch_size do - if inputs_s[1][i] ~= self.vocab.sen_end_token and inputs_s[1][i] ~= self.vocab.null_token then + if bit.band(flags[1][i], nerv.TNN.FC.SEQ_START) == 0 and flags[1][i] > 0 then self.stat.al_sen_start = false end end @@ -198,7 +205,6 @@ function LMReader:get_batch(feeds) if got_new == false then nerv.info("lmseqreader file ends, printing stats...") nerv.printf("al_sen_start:%s\n", tostring(self.stat.al_sen_start)) - return false else return true diff --git a/nerv/examples/lmptb/lmptb/lmutil.lua b/nerv/examples/lmptb/lmptb/lmutil.lua index 6d66d6e..13a5c45 100644 --- a/nerv/examples/lmptb/lmptb/lmutil.lua +++ b/nerv/examples/lmptb/lmptb/lmutil.lua @@ -112,10 +112,17 @@ end --cla:string --w:string --prob:float, the probability -function Result:add(cla, w, prob) - self[cla].logp_all = self[cla].logp_all + math.log10(prob) +function Result:add(cla, w, prob, log10ed) + local lp + if log10ed == true then + lp = prob + else + lp = math.log10(prob) + end + + self[cla].logp_all = self[cla].logp_all + lp if (self.vocab:is_unk_str(w)) then - self[cla].logp_unk = self[cla].logp_unk + math.log10(prob) + self[cla].logp_unk = self[cla].logp_unk + lp self[cla].cn_unk = self[cla].cn_unk + 1 end if (w == self.vocab.sen_end_token) then diff --git a/nerv/examples/lmptb/lmptb/lmvocab.lua b/nerv/examples/lmptb/lmptb/lmvocab.lua index 0e7ef3e..38bb18e 100644 --- a/nerv/examples/lmptb/lmptb/lmvocab.lua +++ b/nerv/examples/lmptb/lmptb/lmvocab.lua @@ -2,8 +2,6 @@ require 'lmptb.lmutil' local Vocab = nerv.class("nerv.LMVocab") -local printf = nerv.printf - local mysplit = function(inputstr, sep) if sep == nil then sep = "%s" @@ -106,7 +104,7 @@ end --fn: string --Add all words in fn to the vocab function Vocab:build_file(fn) - printf("%s Vocab building on file %s...\n", self.log_pre, fn) + nerv.printf("%s Vocab building on file %s...\n", self.log_pre, fn) local file = io.open(fn, "r") while (true) do local list = nerv.LMUtil.read_line(file) @@ -119,7 +117,7 @@ function Vocab:build_file(fn) end end file:close() - printf("%s Building finished, vocab size now is %d.\n", self.log_pre, self:size()) + nerv.printf("%s Building finished, vocab size now is %d.\n", self.log_pre, self:size()) end --[[test diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua index 9bdd5ff..b576834 100644 --- a/nerv/examples/lmptb/lstmlm_ptb_main.lua +++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua @@ -277,7 +277,7 @@ global_conf = { hidden_size = 300, layer_num = 1, chunk_size = 15, - batch_size = 20, + batch_size = 32, max_iter = 35, lr_decay = 1.003, decay_iter = 10, @@ -390,10 +390,10 @@ nerv.LMUtil.wait(2) math.randomseed(1) -local vocab = nerv.LMVocab() +local vocab = nerv.LMVocab(global_conf) global_conf["vocab"] = vocab nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) -global_conf.vocab:build_file(global_conf.vocab_fn, false) +global_conf.vocab:build_file(global_conf.vocab_fn) ppl_rec = {} local final_iter = -1 diff --git a/nerv/examples/lmptb/sample_grulm_ptb_main.lua b/nerv/examples/lmptb/m-tests/lm_sampler_test.lua index 9a13d36..effb2ad 100644 --- a/nerv/examples/lmptb/sample_grulm_ptb_main.lua +++ b/nerv/examples/lmptb/m-tests/lm_sampler_test.lua @@ -134,10 +134,39 @@ function prepare_tnn(global_conf, layerRepo) return tnn end -function prepare_dagL(global_conf, layerRepo) - nerv.printf("%s Generate and initing dagL ...\n", global_conf.sche_log_pre) +function load_net_tnn(global_conf, fn) + prepare_parameters(global_conf, fn) + local layerRepo = prepare_layers(global_conf) + local tnn = prepare_tnn(global_conf, layerRepo) + return tnn +end + +function prepare_sampler(sm_conf) + sm_conf.pr = nerv.ParamRepo() + sm_conf.pr:import({sm_conf.fn_to_sample}, nil, sm_conf) + + local layers = { + ["nerv.GRULayerT"] = { + ["gruL1"] = {{}, {["dim_in"] = {sm_conf.hidden_size, sm_conf.hidden_size}, ["dim_out"] = {sm_conf.hidden_size}, ["pr"] = sm_conf.pr}}, + }, + ["nerv.DropoutLayerT"] = { + ["dropoutL1"] = {{}, {["dim_in"] = {sm_conf.hidden_size}, ["dim_out"] = {sm_conf.hidden_size}}}, + }, + ["nerv.SelectLinearLayer"] = { + ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {sm_conf.hidden_size}, ["vocab"] = sm_conf.vocab, ["pr"] = sm_conf.pr}}, + }, + ["nerv.CombinerLayer"] = { + ["combinerL1"] = {{}, {["dim_in"] = {sm_conf.hidden_size}, ["dim_out"] = {sm_conf.hidden_size, sm_conf.hidden_size}, ["lambda"] = {1}}}, + }, + ["nerv.AffineLayer"] = { + ["outputL"] = {{}, {["dim_in"] = {sm_conf.hidden_size}, ["dim_out"] = {sm_conf.vocab:size()}, ["pr"] = sm_conf.pr}}, + }, + ["nerv.SoftmaxCELayerT"] = { + ["softmaxL"] = {{}, {["dim_in"] = {sm_conf.vocab:size(), sm_conf.vocab:size()}, ["dim_out"] = {1}}}, + }, + } + local layerRepo = nerv.LayerRepo(layers, sm_conf.pr, sm_conf) - --input: input_w, input_w, ... input_w_now, last_activation local connections_t = { ["<input>[1]"] = "selectL1[1]", @@ -151,48 +180,19 @@ function prepare_dagL(global_conf, layerRepo) ["combinerL1[2]"] = "<output>[2]", } - if global_conf.layer_num > 1 then + if sm_conf.layer_num > 1 then nerv.error("multiple layer is currently not supported(not hard to implement though)") end - --[[ - for l = 2, global_conf.layer_num do - table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "gruL"..l.."[1]", 0}) - table.insert(connections_t, {"gruL"..l.."[1]", "combinerL"..l.."[1]", 0}) - table.insert(connections_t, {"combinerL"..l.."[1]", "gruL"..l.."[2]", 1}) - table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) - end - ]]-- - - --[[ - printf("%s printing DAG connections:\n", global_conf.sche_log_pre) - for key, value in pairs(connections_t) do - printf("\t%s->%s\n", key, value) - end - ]]-- - local dagL = nerv.DAGLayerT("dagL", global_conf, {["dim_in"] = {1, global_conf.hidden_size}, - ["dim_out"] = {global_conf.vocab:size(), global_conf.hidden_size}, ["sub_layers"] = layerRepo, + local dagL = nerv.DAGLayerT("dagL", sm_conf, {["dim_in"] = {1, sm_conf.hidden_size}, + ["dim_out"] = {sm_conf.vocab:size(), sm_conf.hidden_size}, ["sub_layers"] = layerRepo, ["connections"] = connections_t }) + + local sampler = nerv.LMSampler(sm_conf) + sampler:load_dagL(dagL) - dagL:init(global_conf.batch_size) - - nerv.printf("%s Initing DAGL end.\n", global_conf.sche_log_pre) - return dagL -end - -function load_net_tnn(global_conf, fn) - prepare_parameters(global_conf, fn) - local layerRepo = prepare_layers(global_conf) - local tnn = prepare_tnn(global_conf, layerRepo) - return tnn -end - -function load_net_dagL(global_conf, fn) - prepare_parameters(global_conf, fn) - local layerRepo = prepare_layers(global_conf) - local dagL = prepare_dagL(global_conf, layerRepo) - return dagL + return sampler end local train_fn, valid_fn, test_fn @@ -240,6 +240,23 @@ global_conf = { fn_to_sample = root_dir .. '/ptb/EXP-nerv/grulm_v1.0h300l1ch15ba32slr0.15wc1e-05dr0.5/params.final', } +sm_conf = { + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + batch_size = 32, + chunk_size = 85, --largest sample sentence length + max_iter = 35, + max_sen_len = 90, + sche_log_pre = "[SAMPLER_S]:", + + timer = global_conf.timer, + fn_to_sample = root_dir .. '/ptb/EXP-nerv/grulm_v1.0h300l1ch15ba32slr0.15wc1e-05dr0.5/params.final', +} + elseif (set == "msr_sc") then data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' @@ -276,15 +293,13 @@ global_conf = { elseif (set == "twitter") then -data_dir = root_dir .. '/twitter_new/DATA' -train_fn = data_dir .. '/twitter.choose2.adds' -valid_fn = data_dir .. '/twitter.valid.adds' -test_fn = data_dir .. '/comm.test.choose-ppl.adds' -vocab_fn = data_dir .. '/twitter.choose.train.vocab' - ---qdata_dir = root_dir .. '/ptb/questionGen/gen' - -global_conf = { + data_dir = root_dir .. '/twitter_new/DATA' + train_fn = data_dir .. '/twitter.choose2.adds' + valid_fn = data_dir .. '/twitter.valid.adds' + test_fn = data_dir .. '/comm.test.choose-ppl.adds' + vocab_fn = data_dir .. '/twitter.choose.train.vocab' + --qdata_dir = root_dir .. '/ptb/questionGen/gen' + global_conf = { lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, cumat_type = nerv.CuMatrixFloat, mmat_type = nerv.MMatrixFloat, @@ -309,7 +324,7 @@ global_conf = { log_w_num = 40000, --give a message when log_w_num words have been processed timer = nerv.Timer(), work_dir_base = root_dir .. '/twitter_new/EXP-nerv/grulm_v1.0' -} + } else @@ -347,15 +362,12 @@ global_conf = { end -lr_half = false --can not be local, to be set by loadstring -start_iter = -1 -start_lr = nil -ppl_last = 100000 commands_str = "sampling" --"train:test" commands = {} -test_iter = -1 ---for testout(question) -q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds" +test_iter = -1 --obselete +random_seed = 1 +sample_num = 10 +out_fn = nil if arg[2] ~= nil then nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) @@ -385,26 +397,27 @@ nerv.LMUtil.wait(2) ]]-- ----------------printing options--------------------------------- -nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre) -for id, value in pairs(global_conf) do +nerv.printf("%s printing sm_conf...\n", sm_conf.sche_log_pre) +for id, value in pairs(sm_conf) do nerv.printf("%s:\t%s\n", id, tostring(value)) end nerv.LMUtil.wait(2) nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) -nerv.printf("lr_half:\t%s\n", tostring(lr_half)) -nerv.printf("start_iter:\t%s\n", tostring(start_iter)) -nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) nerv.printf("commands_str:\t%s\n", commands_str) nerv.printf("test_iter:\t%s\n", tostring(test_iter)) +nerv.printf("random_seed:\t%s\n", tostring(random_seed)) +nerv.printf("sample_num:\t%s\n", tostring(sample_num)) +nerv.printf("out_fn:\t%s\n", tostring(out_fn)) nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) nerv.LMUtil.wait(2) ------------------printing options end------------------------------ -math.randomseed(1) +math.randomseed(random_seed) local vocab = nerv.LMVocab() global_conf["vocab"] = vocab +sm_conf["vocab"] = global_conf.vocab nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) global_conf.vocab:build_file(global_conf.vocab_fn, false) ppl_rec = {} @@ -421,18 +434,34 @@ end --if commands["test"] if commands["sampling"] == 1 then nerv.printf("===SAMPLE===\n") global_conf.sche_log_pre = "[SCHEDULER SAMPLING]:" - local dagL = load_net_dagL(global_conf, global_conf.fn_to_sample) - local sampler = nerv.LMSampler(global_conf) - sampler:load_dagL(dagL) - for k = 1, 5 do - local res = sampler:lm_sample_rnn_dagL(10, {}) + local sampler = prepare_sampler(sm_conf) + local out_fh = nil + if out_fn ~= nil then + out_fh = assert(io.open(out_fn, "w")) + nerv.printf("%s outputing samples to file \"%s\"...\n", global_conf.sche_log_pre, out_fn) + end + for k = 1, sample_num do + local res = sampler:lm_sample_rnn_dagL(1, {}) for i = 1, #res do + if out_fh == nil then nerv.printf("lm_sampler_output_sample: ") end for j = 1, #res[i] do - nerv.printf("%s ", res[i][j].w) + if out_fh == nil then + nerv.printf("%s %f ", res[i][j].w, res[i][j].p) + else + out_fh:write(nerv.sprintf("%s %f ", res[i][j].w, res[i][j].p)) + end + end + if out_fh == nil then + nerv.printf("\n") + else + out_fh:write(nerv.sprintf("\n")) end - nerv.printf("\n") end + if k % 10000 == 0 and out_fh ~= nil then nerv.printf("%s %d sample done\n", global_conf.sche_log_pre, k) end end + + if out_fh ~= nil then out_fh:close() end + nerv.printf("%s complete,bye\n", global_conf.sche_log_pre) --global_conf.dropout_rate = 0 --LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! end --if commands["sampling"] diff --git a/nerv/examples/lmptb/m-tests/lmseqreader_test.lua b/nerv/examples/lmptb/m-tests/lmseqreader_test.lua index 9127559..3f99741 100644 --- a/nerv/examples/lmptb/m-tests/lmseqreader_test.lua +++ b/nerv/examples/lmptb/m-tests/lmseqreader_test.lua @@ -7,7 +7,7 @@ local test_fn = "/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-te --local test_fn = "/home/slhome/txh18/workspace/nerv-project/nerv/examples/lmptb/PTBdata/ptb.train.txt" local vocab = nerv.LMVocab() vocab:build_file(test_fn) -local chunk_size = 20 +local chunk_size = 15 local batch_size = 3 local global_conf = { lrate = 1, wcost = 1e-6, momentum = 0, @@ -30,7 +30,8 @@ local global_conf = { vocab = vocab } -local reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, vocab, {["se_mode"] = true}) +local reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, vocab, + {["se_mode"] = true, ["same_io"] = true}) reader:open_file(test_fn) local feeds = {} feeds.flags_now = {} @@ -40,14 +41,15 @@ for j = 1, chunk_size do feeds.inputs_m[j] = {global_conf.cumat_type(batch_size, 1), global_conf.cumat_type(batch_size, global_conf.vocab:size())} feeds.flags_now[j] = {} end -while (1) do +for k = 1, 5 do local r = reader:get_batch(feeds) if (r == false) then break end for j = 1, chunk_size, 1 do for i = 1, batch_size, 1 do - printf("%s[L(%s)] ", feeds.inputs_s[j][i], feeds.labels_s[j][i]) --vocab:get_word_str(input[i][j]).id + printf("%s[L(%s)]F%d ", feeds.inputs_s[j][i], feeds.labels_s[j][i], feeds.flags_now[j][i]) --vocab:get_word_str(input[i][j]).id end printf("\n") end printf("\n") end +printf("reader.sen_start %s\n", tostring(reader.stat.al_sen_start)) diff --git a/nerv/examples/lmptb/m-tests/some-text b/nerv/examples/lmptb/m-tests/some-text index da4bea9..6756fa0 100644 --- a/nerv/examples/lmptb/m-tests/some-text +++ b/nerv/examples/lmptb/m-tests/some-text @@ -1,4 +1,4 @@ -</s> aa bb cc aa bb cc aa bb cc aa bb cc aa bb cc aa </s> +</s> aa bb cc aa bb cc aa bb cc aa bb cc aa </s> </s> aa bb cc aa bb cc aa bb cc aa </s> </s> bb cc aa bb cc aa bb cc aa </s> </s> aa bb cc aa </s> diff --git a/nerv/examples/lmptb/rnnlm_ptb_main.lua b/nerv/examples/lmptb/rnnlm_ptb_main.lua index dc011fb..a1d9471 100644 --- a/nerv/examples/lmptb/rnnlm_ptb_main.lua +++ b/nerv/examples/lmptb/rnnlm_ptb_main.lua @@ -197,6 +197,43 @@ global_conf = { work_dir_base = root_dir .. '/ptb/EXP-nerv/rnnlm_tnn' } +elseif (set == "twitter") then + +data_dir = root_dir .. '/twitter_new/DATA' +train_fn = data_dir .. '/twitter.choose.adds' +valid_fn = data_dir .. '/twitter.valid.adds' +test_fn = data_dir .. '/comm.test.choose-ppl.adds' +vocab_fn = data_dir .. '/twitter.choose.train.vocab' + +--qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 32, + max_iter = 30, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0.5", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 32, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, |