diff options
Diffstat (limited to 'nerv/examples/lmptb/lmptb')
| -rw-r--r-- | nerv/examples/lmptb/lmptb/layer/affine_recurrent.lua | 93 | ||||
| -rw-r--r-- | nerv/examples/lmptb/lmptb/layer/init.lua | 5 | ||||
| -rw-r--r-- | nerv/examples/lmptb/lmptb/layer/lm_affine_recurrent.lua | 25 | ||||
| -rw-r--r-- | nerv/examples/lmptb/lmptb/lmfeeder.lua | 96 | ||||
| -rw-r--r-- | nerv/examples/lmptb/lmptb/lmutil.lua | 68 | ||||
| -rw-r--r-- | nerv/examples/lmptb/lmptb/lmvocab.lua | 141 |
6 files changed, 428 insertions, 0 deletions
diff --git a/nerv/examples/lmptb/lmptb/layer/affine_recurrent.lua b/nerv/examples/lmptb/lmptb/layer/affine_recurrent.lua new file mode 100644 index 0000000..0a762f0 --- /dev/null +++ b/nerv/examples/lmptb/lmptb/layer/affine_recurrent.lua @@ -0,0 +1,93 @@ +local Recurrent = nerv.class('nerv.AffineRecurrentLayer', 'nerv.Layer') + +--id: string +--global_conf: table +--layer_conf: table +--Get Parameters +function Recurrent:__init(id, global_conf, layer_conf) + self.id = id + self.dim_in = layer_conf.dim_in + self.dim_out = layer_conf.dim_out + self.gconf = global_conf + + self.bp = layer_conf.bp + self.ltp_ih = layer_conf.ltp_ih --from input to hidden + self.ltp_hh = layer_conf.ltp_hh --from hidden to hidden + + self:check_dim_len(2, 1) + self.direct_update = layer_conf.direct_update +end + +--Check parameter +function Recurrent:init(batch_size) + if (self.ltp_ih.trans:ncol() ~= self.bp.trans:ncol() or + self.ltp_hh.trans:ncol() ~= self.bp.trans:ncol()) then + nerv.error("mismatching dimensions of ltp and bp") + end + if (self.dim_in[1] ~= self.ltp_ih.trans:nrow() or + self.dim_in[2] ~= self.ltp_hh.trans:nrow()) then + nerv.error("mismatching dimensions of ltp and input") + end + if (self.dim_out[1] ~= self.bp.trans:ncol()) then + nerv.error("mismatching dimensions of bp and output") + end + + self.ltp_ih_grad = self.ltp_ih.trans:create() + self.ltp_hh_grad = self.ltp_hh.trans:create() + self.ltp_ih:train_init() + self.ltp_hh:train_init() + self.bp:train_init() +end + +function Recurrent:update(bp_err, input, output) + if (self.direct_update == true) then + local ltp_ih = self.ltp_ih.trans + local ltp_hh = self.ltp_hh.trans + local bp = self.bp.trans + local ltc_ih = self.ltc_ih + local ltc_hh = self.ltc_hh + local bc = self.bc + local gconf = self.gconf + -- momentum gain + local mmt_gain = 1.0 / (1.0 - gconf.momentum); + local n = input[1]:nrow() * mmt_gain + -- update corrections (accumulated errors) + self.ltp_ih.correction:mul(input[1], bp_err[1], 1.0, gconf.momentum, 'T', 'N') + self.ltc_hh.correction:mul(input[2], bp_err[1], 1.0, gconf.momentum, 'T', 'N') + self.bp.correction:add(bc, bp_err[1]:colsum(), gconf.momentum, 1.0) + -- perform update + ltp_ih:add(ltp_ih, self.ltp_ih.correction, 1.0, -gconf.lrate / n) + ltp_hh:add(ltp_hh, self.ltp_hh.correction, 1.0, -gconf.lrate / n) + bp:add(bp, self.bp.correction, 1.0, -gconf.lrate / n) + -- weight decay + ltp_ih:add(ltp_ih, ltp_ih, 1.0, -gconf.lrate * gconf.wcost) + ltp_hh:add(ltp_hh, ltp_hh, 1.0, -gconf.lrate * gconf.wcost) + else + self.ltp_ih_grad:mul(input[1], bp_err[1], 1.0, 0.0, 'T', 'N') + self.ltp_ih:update(self.ltp_ih_grad) + self.ltp_hh_grad:mul(input[2], bp_err[1], 1.0, 0.0, 'T', 'N') + self.ltp_hh:update(self.ltp_hh_grad) + self.bp:update(bp_err[1]:colsum()) + end +end + +function Recurrent:propagate(input, output) + output[1]:mul(input[1], self.ltp_ih.trans, 1.0, 0.0, 'N', 'N') + output[1]:mul(input[2], self.ltp_hh.trans, 1.0, 1.0, 'N', 'N') + output[1]:add_row(self.bp.trans, 1.0) +end + +function Recurrent:back_propagate(bp_err, next_bp_err, input, output) + next_bp_err[1]:mul(bp_err[1], self.ltp_ih.trans, 1.0, 0.0, 'N', 'T') + next_bp_err[2]:mul(bp_err[1], self.ltp_hh.trans, 1.0, 0.0, 'N', 'T') + for i = 0, next_bp_err[2]:nrow() - 1 do + for j = 0, next_bp_err[2]:ncol() - 1 do + if (next_bp_err[2][i][j] > 10) then next_bp_err[2][i][j] = 10 end + if (next_bp_err[2][i][j] < -10) then next_bp_err[2][i][j] = -10 end + end + end +end + +function Recurrent:get_params() + return {self.ltp_ih, self.ltp_hh, self.bp} +end diff --git a/nerv/examples/lmptb/lmptb/layer/init.lua b/nerv/examples/lmptb/lmptb/layer/init.lua new file mode 100644 index 0000000..b3b00f6 --- /dev/null +++ b/nerv/examples/lmptb/lmptb/layer/init.lua @@ -0,0 +1,5 @@ +require 'lmptb.layer.affine_recurrent' +require 'lmptb.layer.lm_affine_recurrent' + + + diff --git a/nerv/examples/lmptb/lmptb/layer/lm_affine_recurrent.lua b/nerv/examples/lmptb/lmptb/layer/lm_affine_recurrent.lua new file mode 100644 index 0000000..f1eb4a1 --- /dev/null +++ b/nerv/examples/lmptb/lmptb/layer/lm_affine_recurrent.lua @@ -0,0 +1,25 @@ +local LMRecurrent = nerv.class('nerv.LMAffineRecurrentLayer', 'nerv.AffineRecurrentLayer') --breaks at sentence end, when </s> is met, input will be set to zero + +--id: string +--global_conf: table +--layer_conf: table +--Get Parameters +function LMRecurrent:__init(id, global_conf, layer_conf) + nerv.AffineRecurrentLayer.__init(self, id, global_conf, layer_conf) + self.break_id = layer_conf.break_id --int, breaks recurrent input when the input (word) is break_id + self.independent = layer_conf.independent --bool, whether break +end + +function LMRecurrent:propagate(input, output) + output[1]:mul(input[1], self.ltp_ih.trans, 1.0, 0.0, 'N', 'N') + if (self.independent == true) then + for i = 1, input[1]:nrow() do + if (input[1][i - 1][self.break_id - 1] > 0.1) then --here is sentence break + input[2][i - 1]:fill(0) + end + end + end + output[1]:mul(input[2], self.ltp_hh.trans, 1.0, 1.0, 'N', 'N') + output[1]:add_row(self.bp.trans, 1.0) +end + diff --git a/nerv/examples/lmptb/lmptb/lmfeeder.lua b/nerv/examples/lmptb/lmptb/lmfeeder.lua new file mode 100644 index 0000000..34631bf --- /dev/null +++ b/nerv/examples/lmptb/lmptb/lmfeeder.lua @@ -0,0 +1,96 @@ +require 'lmptb.lmvocab' + +local Feeder = nerv.class("nerv.LMFeeder") + +local printf = nerv.printf + +--global_conf: table +--batch_size: int +--vocab: nerv.LMVocab +function Feeder:__init(global_conf, batch_size, vocab) + self.gconf = global_conf + self.fh = nil --file handle to read, nil means currently no file + self.batch_size = batch_size + self.log_pre = "[LOG]LMFeeder:" + self.vocab = vocab + self.streams = nil +end + +--fn: string +--Initialize all streams +function Feeder:open_file(fn) + if (self.fh ~= nil) then + nerv.error("%s error: in open_file, file handle not nil.") + end + printf("%s opening file %s...\n", self.log_pre, fn) + self.fh = io.open(fn, "r") + self.streams = {} + for i = 1, self.batch_size, 1 do + self.streams[i] = {["store"] = {self.vocab.sen_end_token}, ["head"] = 1, ["tail"] = 1} + end +end + +--id: int +--Refresh stream id, read a line from file +function Feeder:refresh_stream(id) + if (self.streams[id] == nil) then + nerv.error("stream %d does not exit.", id) + end + local st = self.streams[id] + if (st.store[st.head] ~= nil) then return end + if (self.fh == nil) then return end + local list = self.vocab:read_line(self.fh) + if (list == nil) then --file has end + printf("%s file expires, closing.\n", self.log_pre) + self.fh:close() + self.fh = nil + return + end + for i = 1, #list, 1 do + st.tail = st.tail + 1 + st.store[st.tail] = list[i] + end +end + +--Returns: nil/table +--If gets something, return a list of string, vocab.null_token indicates end of string +function Feeder:get_batch() + local got_new = false + local list = {} + for i = 1, self.batch_size, 1 do + self:refresh_stream(i) + local st = self.streams[i] + list[i] = st.store[st.head] + if (list[i] == nil) then list[i] = self.vocab.null_token end + if (list[i] ~= nil and list[i] ~= self.vocab.null_token)then + got_new = true + st.store[st.head] = nil + st.head = st.head + 1 + end + end + if (got_new == false) then + return nil + else + return list + end +end + +--[[ +do + local test_fn = "/home/slhome/txh18/workspace/nerv-project/some-text" + --local test_fn = "/home/slhome/txh18/workspace/nerv-project/nerv/examples/lmptb/PTBdata/ptb.train.txt" + local vocab = nerv.LMVocab() + vocab:build_file(test_fn) + local batch_size = 3 + local feeder = nerv.LMFeeder({}, batch_size, vocab) + feeder:open_file(test_fn) + while (1) do + local list = feeder:get_batch() + if (list == nil) then break end + for i = 1, batch_size, 1 do + printf("%s(%d) ", list[i], vocab:get_word_str(list[i]).id) + end + printf("\n") + end +end +]]-- diff --git a/nerv/examples/lmptb/lmptb/lmutil.lua b/nerv/examples/lmptb/lmptb/lmutil.lua new file mode 100644 index 0000000..c15c637 --- /dev/null +++ b/nerv/examples/lmptb/lmptb/lmutil.lua @@ -0,0 +1,68 @@ +local Util = nerv.class("nerv.LMUtil") + +--list: table, list of string(word) +--vocab: nerv.LMVocab +--ty: nerv.CuMatrix +--Returns: nerv.CuMatrixFloat +--Create a matrix of type 'ty', size #list * vocab:size(). null_word will become a zero vector. +function Util.create_onehot(list, vocab, ty) + local m = ty(#list, vocab:size()) + m:fill(0) + for i = 1, #list, 1 do + --index in matrix starts at 0 + if (list[i] ~= vocab.null_token) then + m[i - 1][vocab:get_word_str(list[i]).id - 1] = 1 + end + end + return m +end + +function Util.wait(sec) + local start = os.time() + repeat until os.time() > start + sec +end + +local Result = nerv.class("nerv.LMResult") + +--global_conf: table +--vocab:nerv.LMVocab +function Result:__init(global_conf, vocab) + self.gconf = global_conf + self.vocab = vocab +end + +--cla:string +--Initialize status of class cla +function Result:init(cla) + self[cla] = {logp_all = 0, logp_unk = 0, cn_w = 0, cn_unk = 0, cn_sen = 0} +end + +--cla:string +--w:string +--prob:float, the probability +function Result:add(cla, w, prob) + self[cla].logp_all = self[cla].logp_all + math.log10(prob) + if (self.vocab:is_unk_str(w)) then + self[cla].logp_unk = self[cla].logp_unk + math.log10(prob) + self[cla].cn_unk = self[cla].cn_unk + 1 + end + if (w == self.vocab.sen_end_token) then + self[cla].cn_sen = self[cla].cn_sen + 1 + else + self[cla].cn_w = self[cla].cn_w + 1 + end +end + +function Result:ppl_net(cla) + local c = self[cla] + return math.pow(10, -(c.logp_all - c.logp_unk) / (c.cn_w - c.cn_unk + c.cn_sen)) +end + +function Result:ppl_all(cla) + local c = self[cla] + return math.pow(10, -(c.logp_all) / (c.cn_w + c.cn_sen)) +end + +function Result:status(cla) + return "LMResult status of " .. cla .. ": " .. "<SEN_CN " .. self[cla].cn_sen .. "> <W_CN " .. self[cla].cn_w .. "> <PPL_NET " .. self:ppl_net(cla) .. "> <PPL_OOV " .. self:ppl_all(cla) .. "> <LOGP " .. self[cla].logp_all .. ">" +end diff --git a/nerv/examples/lmptb/lmptb/lmvocab.lua b/nerv/examples/lmptb/lmptb/lmvocab.lua new file mode 100644 index 0000000..3d256c0 --- /dev/null +++ b/nerv/examples/lmptb/lmptb/lmvocab.lua @@ -0,0 +1,141 @@ +local Vocab = nerv.class("nerv.LMVocab") + +local printf = nerv.printf + +local mysplit = function(inputstr, sep) + if sep == nil then + sep = "%s" + end + local t={} ; i=1 + for str in string.gmatch(inputstr, "([^"..sep.."]+)") do + t[i] = str + i = i + 1 + end + return t +end + +function Vocab:__init(global_conf) + self.gconf = global_conf + self.sen_end_token = "</s>" + self.unk_token = "<unk>" + self.null_token = "<null>" --indicating end of stream(in feeder) + self.log_pre = "[LOG]LMVocab:" + self.map_str = {} --map from str to word_entry + self.map_id = {} --map from id to word_entry + + self:add_word(self.sen_end_token) + self:add_word(self.unk_token) +end + +--id: int +--w_str: string +--Returns: table +function Vocab:new_word_entry(id, w_str) + return { ["id"] = id, + ["str"] = w_str, + ["cnt"] = 0, + } +end + +--Returns: int +function Vocab:size() + return #self.map_id +end + +--w_str: string +--if w_str is not in vocab, then add it in, if it is already in, do nothing +function Vocab:add_word(w_str) + if (self.map_str[w_str] ~= nil) then + return + end + local e = self:new_word_entry(self:size() + 1, w_str) + self.map_id[self:size() + 1] = e + self.map_str[w_str] = e +end + +--Returns: table, the entry of the unk +function Vocab:get_unk_entry() + if (self.map_str[self.unk_token] == nil) then + nerv.error("unk entry not found.") + end + return self.map_str[self.unk_token] +end + +--Returns: table, the entry of sentence end +function Vocab:get_sen_entry() + if (self.map_str[self.sen_end_token] == nil) then + nerv.error("sen end token not found") + end + return self.map_str[self.sen_end_token] +end + +function Vocab:is_unk_str(w) + if (key == self.null_token) then + nerv.error("Vocab:get_word_str is called by the null token") + end + if (w == self.unk_token or self.map_str[w] == nil) then + return true + else + return false + end +end + +--key: string +--Returns: table, the word_entry of this key +function Vocab:get_word_str(key) + if (self.map_str[key] == nil) then + return self:get_unk_entry() + end + if (key == self.null_token) then + nerv.error("Vocab:get_word_str is called by the null token") + end + return self.map_str[key] +end + +--key: int +--Returns: table +function Vocab:get_word_id(key) + if (self.map_id[key] == nil) then + nerv.error("id key %d does not exist.", key) + end + return self.map_id(key) +end + +--fh: file_handle +--Returns: a list of tokens(string) in the line, if there is no "</s>" at the end, the function will at it, if nothing to read, returns nil +function Vocab:read_line(fh) + local l_str = fh:read("*line") + if (l_str == nil) then return nil end + local list = mysplit(l_str) + if (list[(#list)] ~= self.sen_end_token) then + list[#list + 1] = self.sen_end_token + end + return list +end + +--fn: string +--Add all words in fn to the vocab +function Vocab:build_file(fn) + printf("%s Vocab building on file %s...\n", self.log_pre, fn) + local file = io.open(fn, "r") + while (true) do + local list = self:read_line(file) + if (list == nil) then + break + else + for i = 1, #list, 1 do + self:add_word(list[i]) + end + end + end + file:close() + printf("%s Building finished, vocab size now is %d.\n", self.log_pre, self:size()) +end + +--[[test +do + local test_fn = "/home/slhome/txh18/workspace/nerv-project/some-text" + local vocab = nerv.LMVocab() + vocab:build_file(test_fn) +end +]]-- |