diff options
Diffstat (limited to 'nerv/examples/lmptb/lmptb/lmvocab.lua')
| -rw-r--r-- | nerv/examples/lmptb/lmptb/lmvocab.lua | 18 |
1 files changed, 4 insertions, 14 deletions
diff --git a/nerv/examples/lmptb/lmptb/lmvocab.lua b/nerv/examples/lmptb/lmptb/lmvocab.lua index 3d256c0..0e7ef3e 100644 --- a/nerv/examples/lmptb/lmptb/lmvocab.lua +++ b/nerv/examples/lmptb/lmptb/lmvocab.lua @@ -1,3 +1,5 @@ +require 'lmptb.lmutil' + local Vocab = nerv.class("nerv.LMVocab") local printf = nerv.printf @@ -98,19 +100,7 @@ function Vocab:get_word_id(key) if (self.map_id[key] == nil) then nerv.error("id key %d does not exist.", key) end - return self.map_id(key) -end - ---fh: file_handle ---Returns: a list of tokens(string) in the line, if there is no "</s>" at the end, the function will at it, if nothing to read, returns nil -function Vocab:read_line(fh) - local l_str = fh:read("*line") - if (l_str == nil) then return nil end - local list = mysplit(l_str) - if (list[(#list)] ~= self.sen_end_token) then - list[#list + 1] = self.sen_end_token - end - return list + return self.map_id[key] end --fn: string @@ -119,7 +109,7 @@ function Vocab:build_file(fn) printf("%s Vocab building on file %s...\n", self.log_pre, fn) local file = io.open(fn, "r") while (true) do - local list = self:read_line(file) + local list = nerv.LMUtil.read_line(file) if (list == nil) then break else |