Merge pull request #21 from cloudygoose/txh18/rnnlm

Txh18/rnnlm new changes to lm side

1 files changed, 4 insertions, 14 deletions

diff --git a/nerv/examples/lmptb/lmptb/lmvocab.lua b/nerv/examples/lmptb/lmptb/lmvocab.lua
index 3d256c0..0e7ef3e 100644
--- a/nerv/examples/lmptb/lmptb/lmvocab.lua
+++ b/nerv/examples/lmptb/lmptb/lmvocab.lua
@@ -1,3 +1,5 @@
+require 'lmptb.lmutil'
+
 local Vocab = nerv.class("nerv.LMVocab")
 
 local printf = nerv.printf
@@ -98,19 +100,7 @@ function Vocab:get_word_id(key)
     if (self.map_id[key] == nil) then
         nerv.error("id key %d does not exist.", key) 
     end
-    return self.map_id(key)
-end
-
---fh: file_handle
---Returns: a list of tokens(string) in the line, if there is no "</s>" at the end, the function will at it, if nothing to read, returns nil
-function Vocab:read_line(fh)
-    local l_str = fh:read("*line")
-    if (l_str == nil) then return nil end
-    local list = mysplit(l_str)
-    if (list[(#list)] ~= self.sen_end_token) then
-        list[#list + 1] = self.sen_end_token
-    end
-    return list
+    return self.map_id[key]
 end
 
 --fn: string
@@ -119,7 +109,7 @@ function Vocab:build_file(fn)
     printf("%s Vocab building on file %s...\n", self.log_pre, fn)
     local file = io.open(fn, "r")
     while (true) do
-        local list = self:read_line(file)
+        local list = nerv.LMUtil.read_line(file)
         if (list == nil) then
             break
         else