added twitter, added bilstmlm script, todo: test bilstmlm

1 files changed, 43 insertions, 4 deletions

diff --git a/nerv/examples/lmptb/lstmlm_ptb_main.lua b/nerv/examples/lmptb/lstmlm_ptb_main.lua
index a2dcbba..887993d 100644
--- a/nerv/examples/lmptb/lstmlm_ptb_main.lua
+++ b/nerv/examples/lmptb/lstmlm_ptb_main.lua
@@ -257,12 +257,50 @@ global_conf = {
     work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test'
 }
 
+elseif (set == "twitter") then
+
+root_dir = '/home/slhome/txh18/workspace'
+data_dir = root_dir .. '/twitter_new/DATA'
+train_fn = data_dir .. '/twitter.choose.adds'
+valid_fn = data_dir .. '/twitter.valid.adds'
+test_fn = data_dir .. '/comm.test.choose-ppl.adds'
+vocab_fn = data_dir .. '/twitter.choose.train.vocab'
+
+--qdata_dir = root_dir .. '/ptb/questionGen/gen'
+
+global_conf = {
+    lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5,
+    cumat_type = nerv.CuMatrixFloat,
+    mmat_type = nerv.MMatrixFloat,
+    nn_act_default = 0, 
+
+    hidden_size = 300,
+    layer_num = 1,
+    chunk_size = 15,
+    batch_size = 20, 
+    max_iter = 35,
+    lr_decay = 1.003,
+    decay_iter = 10,
+    param_random = function() return (math.random() / 5 - 0.1) end,
+    dropout_str = "0",
+
+    train_fn = train_fn,
+    valid_fn = valid_fn,
+    test_fn = test_fn,
+    vocab_fn = vocab_fn,
+    max_sen_len = 90,
+    sche_log_pre = "[SCHEDULER]:",
+    log_w_num = 40000, --give a message when log_w_num words have been processed
+    timer = nerv.Timer(),
+    work_dir_base = root_dir .. '/twitter_new/EXP-nerv/lstmlm_v1.0'
+}
+
 else
 
-valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'
-train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'
-test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'
-vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'
+valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn'
+train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn'
+test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn'
+vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn'
 
 global_conf = {
     lrate = 0.01, wcost = 1e-5, momentum = 0,
@@ -281,6 +319,7 @@ global_conf = {
     train_fn = train_fn,
     valid_fn = valid_fn,
     test_fn = test_fn,
+    max_sen_len = 80,
     lr_decay = 1.003,
     decay_iter = 10,
     vocab_fn = vocab_fn,