+require 'lmptb.lmvocab'

+require 'lmptb.lmfeeder'

+require 'lmptb.lmutil'

+require 'lmptb.layer.init'

+--require 'tnn.init'

+require 'lmptb.lmseqreader'

+require 'lm_trainer'

+

+--[[global function rename]]--

+--local printf = nerv.printf

+local LMTrainer = nerv.LMTrainer

+--[[global function rename ends]]--

+

+--global_conf: table

+--first_time: bool

+--Returns: a ParamRepo

+function prepare_parameters(global_conf, iter)

+ nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre)

+

+ global_conf.paramRepo = nerv.ParamRepo()

+ local paramRepo = global_conf.paramRepo

+

+ if iter == -1 then --first time

+ nerv.printf("%s first time, prepare some pre-set parameters, and leaving other parameters to auto-generation...\n", global_conf.sche_log_pre)

+ local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w')

+ f:close()

+ --[[

+ ltp_ih = nerv.LinearTransParam("ltp_ih", global_conf)

+ ltp_ih.trans = global_conf.cumat_type(global_conf.vocab:size(), global_conf.hidden_size) --index 0 is for zero, others correspond to vocab index(starting from 1)

+ ltp_ih.trans:generate(global_conf.param_random)

+

+ ltp_hh = nerv.LinearTransParam("ltp_hh", global_conf)

+ ltp_hh.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.hidden_size)

+ ltp_hh.trans:generate(global_conf.param_random)

+

+ --ltp_ho = nerv.LinearTransParam("ltp_ho", global_conf)

+ --ltp_ho.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.vocab:size())

+ --ltp_ho.trans:generate(global_conf.param_random)

+

+ bp_h = nerv.BiasParam("bp_h", global_conf)

+ bp_h.trans = global_conf.cumat_type(1, global_conf.hidden_size)

+ bp_h.trans:generate(global_conf.param_random)

+

+ --bp_o = nerv.BiasParam("bp_o", global_conf)

+ --bp_o.trans = global_conf.cumat_type(1, global_conf.vocab:size())

+ --bp_o.trans:generate(global_conf.param_random)

+

+ local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w')

+ f:write_chunk(ltp_ih)

+ f:write_chunk(ltp_hh)

+ --f:write_chunk(ltp_ho)

+ f:write_chunk(bp_h)

+ --f:write_chunk(bp_o)

+ f:close()

+ ]]--

+ return nil

+ end

+

+ nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, global_conf.param_fn .. '.' .. tostring(iter))

+ paramRepo:import({global_conf.param_fn .. '.' .. tostring(iter)}, nil, global_conf)

+

+ nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre)

+

+ return nil

+end

+

+--global_conf: table

+--Returns: nerv.LayerRepo

+function prepare_layers(global_conf)

+ nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre)

+

+ local pr = global_conf.paramRepo

+

+ local du = false

+

+ local layers = {

+ ["nerv.LSTMLayerT"] = {

+ ["lstmFL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}},

+ ["lstmRL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}},

+ },

+

+ ["nerv.DropoutLayerT"] = {

+ ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}},

+ },

+

+ ["nerv.SelectLinearLayer"] = {

+ ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}},

+ },

+

+ ["nerv.CombinerLayer"] = {

+ ["combinerXL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}},

+ ["combinerHFL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}},

+ ["combinerHRL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}},

+ },

+

+ ["nerv.AffineLayer"] = {

+ ["biAffineL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr, ["lambda"] = {1, 1}}},

+ ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}},

+ },

+

+ ["nerv.TanhLayer"] = {

+ ["biTanhL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}},

+ },

+

+ ["nerv.SoftmaxCELayerT"] = {

+ ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}},

+ },

+ }

+

+ if global_conf.layer_num > 1 then

+ nerv.error("this script currently do not support more than one layer")

+ end

+ --[[

+ for l = 2, global_conf.layer_num do

+ layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}

+ layers["nerv.LSTMLayerT"]["lstmL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["pr"] = pr}}

+ layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}

+ end

+ ]]--

+

+ local layerRepo = nerv.LayerRepo(layers, pr, global_conf)

+ nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre)

+ return layerRepo

+end

+

+--global_conf: table

+--layerRepo: nerv.LayerRepo

+--Returns: a nerv.TNN

+function prepare_tnn(global_conf, layerRepo)

+ nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre)

+

+ --input: input_w, input_w, ... input_w_now, last_activation

+ local connections_t = {

+ {"<input>[1]", "selectL1[1]", 0},

+

+ --{"selectL1[1]", "recurrentL1[1]", 0},

+ --{"recurrentL1[1]", "sigmoidL1[1]", 0},

+ --{"sigmoidL1[1]", "combinerL1[1]", 0},

+ --{"combinerL1[1]", "recurrentL1[2]", 1},

+

+ {"selectL1[1]", "combinerXL1[1]", 0},

+ {"combinerXL1[1]", "lstmFL1[1]", 0},

+ {"lstmFL1[1]", "combinerHFL1[1]", 0},

+ {"combinerHFL1[1]", "lstmFL1[2]", 1},

+ {"lstmFL1[2]", "lstmFL1[3]", 1},

+ {"combinerXL1[2]", "lstmRL1[1]", 0},

+ {"lstmRL1[1]", "combinerHRL1[1]", 0},

+ {"combinerHRL1[1]", "lstmRL1[2]", -1},

+ {"lstmRL1[2]", "lstmRL1[3]", -1},

+ {"combinerHFL1[2]", "biAffineL1[1]", 0},

+ {"combinerHRL1[2]", "biAffineL1[2]", 0},

+ {"biAffineL1[1]", "biTanhL1[1]", 0},

+ {"biTanhL1[1]", "dropoutL1[1]", 0},

+

+ {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0},

+ {"outputL[1]", "softmaxL[1]", 0},

+ {"<input>[2]", "softmaxL[2]", 0},

+ {"softmaxL[1]", "<output>[1]", 0}

+ }

+

+ --[[

+ for l = 2, global_conf.layer_num do

+ table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "lstmL"..l.."[1]", 0})

+ table.insert(connections_t, {"lstmL"..l.."[2]", "lstmL"..l.."[3]", 1})

+ table.insert(connections_t, {"lstmL"..l.."[1]", "combinerL"..l.."[1]", 0})

+ table.insert(connections_t, {"combinerL"..l.."[1]", "lstmL"..l.."[2]", 1})

+ table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0})

+ end

+ ]]--

+

+ --[[

+ printf("%s printing DAG connections:\n", global_conf.sche_log_pre)

+ for key, value in pairs(connections_t) do

+ printf("\t%s->%s\n", key, value)

+ end

+ ]]--

+

+ local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()},

+ ["dim_out"] = {1}, ["sub_layers"] = layerRepo,

+ ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t,

+ })

+

+ tnn:init(global_conf.batch_size, global_conf.chunk_size)

+

+ nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre)

+ return tnn

+end

+

+function load_net(global_conf, next_iter)

+ prepare_parameters(global_conf, next_iter)

+ local layerRepo = prepare_layers(global_conf)

+ local tnn = prepare_tnn(global_conf, layerRepo)

+ return tnn

+end

+

+local train_fn, valid_fn, test_fn

+global_conf = {}

+local set = arg[1] --"test"

+

+if (set == "ptb") then

+

+root_dir = '/home/slhome/txh18/workspace'

+data_dir = root_dir .. '/ptb/DATA'

+train_fn = data_dir .. '/ptb.train.txt.adds'

+valid_fn = data_dir .. '/ptb.valid.txt.adds'

+test_fn = data_dir .. '/ptb.test.txt.adds'

+vocab_fn = data_dir .. '/vocab'

+

+qdata_dir = root_dir .. '/ptb/questionGen/gen'

+

+global_conf = {

+ lrate = 0.015, wcost = 1e-5, momentum = 0, clip_t = 5,

+ cumat_type = nerv.CuMatrixFloat,

+ mmat_type = nerv.MMatrixFloat,

+ nn_act_default = 0,

+

+ hidden_size = 300,

+ layer_num = 1,

+ chunk_size = 90,

+ batch_size = 20,

+ max_iter = 35,

+ lr_decay = 1.003,

+ decay_iter = 10,

+ param_random = function() return (math.random() / 5 - 0.1) end,

+ dropout_str = "0",

+

+ train_fn = train_fn,

+ valid_fn = valid_fn,

+ test_fn = test_fn,

+ vocab_fn = vocab_fn,

+ max_sen_len = 90,

+ sche_log_pre = "[SCHEDULER]:",

+ log_w_num = 40000, --give a message when log_w_num words have been processed

+ timer = nerv.Timer(),

+ work_dir_base = '/home/slhome/txh18/workspace/ptb/EXP-nerv/bilstmlm_v1.0'

+}

+

+elseif (set == "msr_sc") then

+

+data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2'

+train_fn = data_dir .. '/normed_all.sf.len60.adds.train'

+valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev'

+test_fn = data_dir .. '/answer_normed.adds'

+vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab'

+

+global_conf = {

+ lrate = 1, wcost = 1e-6, momentum = 0,

+ cumat_type = nerv.CuMatrixFloat,

+ mmat_type = nerv.MMatrixFloat,

+ nn_act_default = 0,

+

+ hidden_size = 300,

+ layer_num = 1,

+ chunk_size = 15,

+ batch_size = 10,

+ max_iter = 30,

+ decay_iter = 10,

+ lr_decay = 1.003,

+ param_random = function() return (math.random() / 5 - 0.1) end,

+ dropout_str = "0",

+

+ train_fn = train_fn,

+ valid_fn = valid_fn,

+ test_fn = test_fn,

+ vocab_fn = vocab_fn,

+ sche_log_pre = "[SCHEDULER]:",

+ log_w_num = 400000, --give a message when log_w_num words have been processed

+ timer = nerv.Timer(),

+ work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test'

+}

+

+elseif (set == "twitter") then

+

+root_dir = '/home/slhome/txh18/workspace'

+data_dir = root_dir .. '/twitter_new/DATA'

+train_fn = data_dir .. '/twitter.choose.adds'

+valid_fn = data_dir .. '/twitter.valid.adds'

+test_fn = data_dir .. '/comm.test.choose-ppl.adds'

+vocab_fn = data_dir .. '/twitter.choose.train.vocab'

+

+--qdata_dir = root_dir .. '/ptb/questionGen/gen'

+

+global_conf = {

+ lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5,

+ cumat_type = nerv.CuMatrixFloat,

+ mmat_type = nerv.MMatrixFloat,

+ nn_act_default = 0,

+

+ hidden_size = 300,

+ layer_num = 1,

+ chunk_size = 15,

+ batch_size = 20,

+ max_iter = 35,

+ lr_decay = 1.003,

+ decay_iter = 10,

+ param_random = function() return (math.random() / 5 - 0.1) end,

+ dropout_str = "0",

+

+ train_fn = train_fn,

+ valid_fn = valid_fn,

+ test_fn = test_fn,

+ vocab_fn = vocab_fn,

+ max_sen_len = 90,

+ sche_log_pre = "[SCHEDULER]:",

+ log_w_num = 40000, --give a message when log_w_num words have been processed

+ timer = nerv.Timer(),

+ work_dir_base = root_dir .. '/twitter_new/EXP-nerv/bilstmlm_v1.0'

+}

+

+else

+

+valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'

+train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'

+test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'

+vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'

+

+global_conf = {

+ lrate = 0.01, wcost = 1e-5, momentum = 0,

+ cumat_type = nerv.CuMatrixFloat,

+ mmat_type = nerv.MMatrixFloat,

+ nn_act_default = 0,

+

+ hidden_size = 20,

+ layer_num = 1,

+ chunk_size = 20,

+ batch_size = 10,

+ max_iter = 2,

+ param_random = function() return (math.random() / 5 - 0.1) end,

+ dropout_str = "0",

+

+ train_fn = train_fn,

+ valid_fn = valid_fn,

+ test_fn = test_fn,

+ max_sen_len = 80,

+ lr_decay = 1.003,

+ decay_iter = 10,

+ vocab_fn = vocab_fn,

+ sche_log_pre = "[SCHEDULER]:",

+ log_w_num = 10, --give a message when log_w_num words have been processed

+ timer = nerv.Timer(),

+ work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_bilstmlm_test'

+}

+

+end

+

+lr_half = false --can not be local, to be set by loadstring

+start_iter = -1

+start_lr = global_conf.lrate

+ppl_last = 100000

+commands_str = "train:test"

+commands = {}

+test_iter = -1

+

+--for testout(question)

+q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds"

+

+if arg[2] ~= nil then

+ nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2])

+ loadstring(arg[2])()

+ nerv.LMUtil.wait(0.5)

+else

+ nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre)

+end

+

+global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str

+global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf'

+global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak'

+global_conf.param_fn = global_conf.work_dir .. "/params"

+global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str)

+global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time())

+global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-')

+commands = nerv.SUtil.parse_commands_set(commands_str)

+

+global_conf.lrate = start_lr --starting lr can be set by user(arg[2])

+

+nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir)

+nerv.LMUtil.wait(2)

+os.execute("mkdir -p "..global_conf.work_dir)

+os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf)

+

+--redirecting log outputs!

+nerv.SUtil.log_redirect(global_conf.log_fn)

+nerv.LMUtil.wait(2)

+

+----------------printing options---------------------------------

+nerv.printf("%s printing global_conf...\n", global_conf.sche_log_pre)

+for id, value in pairs(global_conf) do

+ nerv.printf("%s:\t%s\n", id, tostring(value))

+end

+nerv.LMUtil.wait(2)

+

+nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre)

+nerv.printf("lr_half:\t%s\n", tostring(lr_half))

+nerv.printf("start_iter:\t%s\n", tostring(start_iter))

+nerv.printf("ppl_last:\t%s\n", tostring(ppl_last))

+nerv.printf("commands_str:\t%s\n", commands_str)

+nerv.printf("test_iter:\t%s\n", tostring(test_iter))

+nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre)

+nerv.LMUtil.wait(2)

+------------------printing options end------------------------------

+

+math.randomseed(1)

+

+local vocab = nerv.LMVocab()

+global_conf["vocab"] = vocab

+nerv.printf("%s building vocab...\n", global_conf.sche_log_pre)

+global_conf.vocab:build_file(global_conf.vocab_fn, false)

+ppl_rec = {}

+

+local final_iter = -1

+if commands["train"] == 1 then

+ if start_iter == -1 then

+ prepare_parameters(global_conf, -1) --write pre_generated params to param.0 file

+ end

+

+ if start_iter == -1 or start_iter == 0 then

+ nerv.printf("===INITIAL VALIDATION===\n")

+ local tnn = load_net(global_conf, 0)

+ global_conf.paramRepo = tnn:get_params() --get auto-generted params

+ global_conf.paramRepo:export(global_conf.param_fn .. '.0', nil) --some parameters are auto-generated, saved again to param.0 file

+ global_conf.dropout_rate = 0

+ local result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.valid_fn, tnn, false) --false update!

+ nerv.LMUtil.wait(1)

+ ppl_rec[0] = {}

+ ppl_rec[0].valid = result:ppl_all("birnn")

+ ppl_last = ppl_rec[0].valid

+ ppl_rec[0].train = 0

+ ppl_rec[0].test = 0

+ ppl_rec[0].lr = 0

+

+ start_iter = 1

+

+ nerv.printf("\n")

+ end

+

+ for iter = start_iter, global_conf.max_iter, 1 do

+ final_iter = iter --for final testing

+ global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:"

+ tnn = load_net(global_conf, iter - 1)

+ nerv.printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate)

+ global_conf.dropout_rate = nerv.SUtil.sche_get(global_conf.dropout_list, iter)

+ result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.train_fn_shuf, tnn, true) --true update!

+ global_conf.dropout_rate = 0

+ ppl_rec[iter] = {}

+ ppl_rec[iter].train = result:ppl_all("birnn")

+ --shuffling training file

+ nerv.printf("%s shuffling training file\n", global_conf.sche_log_pre)

+ os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak)

+ os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf)

+ nerv.printf("===PEEK ON TEST %d===\n", iter)

+ result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.test_fn, tnn, false) --false update!

+ ppl_rec[iter].test = result:ppl_all("birnn")

+ nerv.printf("===VALIDATION %d===\n", iter)

+ result = LMTrainer.lm_process_file_birnn(global_conf, global_conf.valid_fn, tnn, false) --false update!

+ ppl_rec[iter].valid = result:ppl_all("birnn")

+ ppl_rec[iter].lr = global_conf.lrate

+ if ((ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true) and iter > global_conf.decay_iter) then

+ global_conf.lrate = (global_conf.lrate * 0.6)

+ end

+ if ppl_rec[iter].valid < ppl_last then

+ nerv.printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter)

+ global_conf.paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil)

+ else

+ nerv.printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre)

+ os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter))

+ end

+ if ppl_last / ppl_rec[iter].valid < global_conf.lr_decay or lr_half == true then

+ lr_half = true

+ end

+ if ppl_rec[iter].valid < ppl_last then

+ ppl_last = ppl_rec[iter].valid

+ end

+ nerv.printf("\n")

+ nerv.LMUtil.wait(2)

+ end

+ nerv.info("saving final nn to param.final")

+ os.execute('cp ' .. global_conf.param_fn .. '.' .. tostring(final_iter) .. ' ' .. global_conf.param_fn .. '.final')

+

+ nerv.printf("===VALIDATION PPL record===\n")

+ for i, _ in pairs(ppl_rec) do

+ nerv.printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test)

+ end

+ nerv.printf("\n")

+end --if commands["train"]

+

+if commands["test"] == 1 then

+ nerv.printf("===FINAL TEST===\n")

+ global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:"

+ if final_iter ~= -1 and test_iter == -1 then

+ test_iter = final_iter

+ end

+ if test_iter == -1 then

+ test_iter = "final"

+ end

+ tnn = load_net(global_conf, test_iter)

+ global_conf.dropout_rate = 0

+ LMTrainer.lm_process_file_birnn(global_conf, global_conf.test_fn, tnn, false) --false update!

+end --if commands["test"]

+

+if commands["testout"] == 1 then

+ nerv.printf("===TEST OUT===\n")

+ nerv.printf("q_file:\t%s\n", q_file)

+ local q_fn = q_file --qdata_dir .. '/' .. q_file

+ global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:"

+ if final_iter ~= -1 and test_iter == -1 then

+ test_iter = final_iter

+ end

+ if test_iter == -1 then

+ test_iter = "final"

+ end

+ tnn = load_net(global_conf, test_iter)

+ global_conf.dropout_rate = 0

+ LMTrainer.lm_process_file_birnn(global_conf, q_fn, tnn, false,

+ {["one_sen_report"] = true}) --false update!

+end --if commands["testout"]

+

+

+function LMTrainer.lm_process_file_rnn(global_conf, fn, tnn, do_train, p_conf)

+ if p_conf == nil then

+ p_conf = {}

+ end

+ local reader

+ local r_conf = {}

+ local chunk_size, batch_size

+ if p_conf.one_sen_report == true then --report log prob one by one sentence

+ if do_train == true then

+ nerv.warning("LMTrainer.lm_process_file_rnn: warning, one_sen_report is true while do_train is also true, strange")

+ end

+ nerv.printf("lm_process_file_rnn: one_sen report mode, set batch_size to 1 and chunk_size to max_sen_len(%d)\n",

+ global_conf.max_sen_len)

+ batch_size = 1

+ chunk_size = global_conf.max_sen_len

+ r_conf["se_mode"] = true

+ else

+ batch_size = global_conf.batch_size

+ chunk_size = global_conf.chunk_size

+ end

+

+ reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, global_conf.vocab, r_conf)

+

+ tnn:init(batch_size, chunk_size)

+ local neto_bakm = global_conf.mmat_type(batch_size, 1) --space backup matrix for network output

+

+ nerv.info("LMTrainer.lm_process_file_rnn: begin processing...")

+ for t = 1, chunk_size do

+ for i = 1, batch_size do

+ local sen_logp = {}

+ for t = 1, chunk_size, 1 do

+ for i = 1, batch_size, 1 do

+ if sen_logp[i] == nil then

+ sen_logp[i] = 0

+ end

+ sen_logp[i] = sen_logp[i] + neto_bakm[i - 1][0]

+ if p_conf.one_sen_report == true then

+ for i = 1, batch_size do

+ nerv.printf("LMTrainer.lm_process_file_rnn: one_sen_report_output, %f\n", sen_logp[i])

+ end

+ end

+

+--Returns: LMResult

+function LMTrainer.lm_process_file_birnn(global_conf, fn, tnn, do_train, p_conf)

+ if p_conf == nil then

+ p_conf = {}

+ end

+ local reader

+ local chunk_size, batch_size

+ local r_conf = {["se_mode"] = true}

+ if p_conf.one_sen_report == true then --report log prob one by one sentence

+ if do_train == true then

+ nerv.warning("LMTrainer.lm_process_file_birnn: warning, one_sen_report is true while do_train is also true, strange")

+ end

+ nerv.printf("lm_process_file_birnn: one_sen report mode, set batch_size to 1 and chunk_size to max_sen_len(%d)\n",

+ global_conf.max_sen_len)

+ batch_size = 1

+ chunk_size = global_conf.max_sen_len

+ else

+ batch_size = global_conf.batch_size

+ chunk_size = global_conf.chunk_size

+ end

+

+ reader = nerv.LMSeqReader(global_conf, batch_size, chunk_size, global_conf.vocab, r_conf)

+ reader:open_file(fn)

+

+ local result = nerv.LMResult(global_conf, global_conf.vocab)

+ result:init("birnn")

+ if global_conf.dropout_rate ~= nil then

+ nerv.info("LMTrainer.lm_process_file_birnn: dropout_rate is %f", global_conf.dropout_rate)

+ end

+

+ global_conf.timer:flush()

+ tnn:init(batch_size, chunk_size)

+ tnn:flush_all() --caution: will also flush the inputs from the reader!

+

+ local next_log_wcn = global_conf.log_w_num

+ local neto_bakm = global_conf.mmat_type(batch_size, 1) --space backup matrix for network output

+

+ nerv.info("LMTrainer.lm_process_file_birnn: begin processing...")

+

+ while (1) do

+ global_conf.timer:tic('most_out_loop_lmprocessfile')

+

+ local r, feeds

+ global_conf.timer:tic('tnn_beforeprocess')

+ r, feeds = tnn:getfeed_from_reader(reader)

+ if r == false then

+ break

+ end

+

+ for t = 1, chunk_size do

+ tnn.err_inputs_m[t][1]:fill(1)

+ for i = 1, batch_size do

+ if bit.band(feeds.flags_now[t][i], nerv.TNN.FC.HAS_LABEL) == 0 then

+ tnn.err_inputs_m[t][1][i - 1][0] = 0

+ end

+ end

+ end

+ global_conf.timer:toc('tnn_beforeprocess')

+

+ --[[

+ for j = 1, global_conf.chunk_size, 1 do

+ for i = 1, global_conf.batch_size, 1 do

+ printf("%s[L(%s)] ", feeds.inputs_s[j][i], feeds.labels_s[j][i]) --vocab:get_word_str(input[i][j]).id

+ end

+ printf("\n")

+ end

+ printf("\n")

+ ]]--

+

+ tnn:net_propagate()

+

+ if do_train == true then

+ tnn:net_backpropagate(false)

+ tnn:net_backpropagate(true)

+ end

+

+ global_conf.timer:tic('tnn_afterprocess')

+ local sen_logp = {}

+ for t = 1, chunk_size, 1 do

+ tnn.outputs_m[t][1]:copy_toh(neto_bakm)

+ for i = 1, batch_size, 1 do

+ if (feeds.labels_s[t][i] ~= global_conf.vocab.null_token) then

+ result:add("birnn", feeds.labels_s[t][i], math.exp(neto_bakm[i - 1][0]))

+ if sen_logp[i] == nil then

+ sen_logp[i] = 0

+ end

+ sen_logp[i] = sen_logp[i] + neto_bakm[i - 1][0]

+ end

+ end

+ end

+ if p_conf.one_sen_report == true then

+ for i = 1, batch_size do

+ nerv.printf("LMTrainer.lm_process_file_birnn: one_sen_report_output, %f\n", sen_logp[i])

+ end

+ end

+

+ --tnn:move_right_to_nextmb({0}) --do not need history for bi directional model

+ global_conf.timer:toc('tnn_afterprocess')

+

+ global_conf.timer:toc('most_out_loop_lmprocessfile')

+

+ --print log

+ if result["birnn"].cn_w > next_log_wcn then

+ next_log_wcn = next_log_wcn + global_conf.log_w_num

+ nerv.printf("%s %d words processed %s.\n", global_conf.sche_log_pre, result["birnn"].cn_w, os.date())

+ nerv.printf("\t%s log prob per sample :%f.\n", global_conf.sche_log_pre, result:logp_sample("birnn"))

+ for key, value in pairs(global_conf.timer.rec) do

+ nerv.printf("\t [global_conf.timer]: time spent on %s:%.5f clock time\n", key, value)

+ end

+ global_conf.timer:flush()

+ nerv.LMUtil.wait(0.1)

+ end

+

+ --[[

+ for t = 1, global_conf.chunk_size do

+ print(tnn.outputs_m[t][1])

+ end

+ ]]--

+

+ collectgarbage("collect")

+

+ --break --debug

+ end

+

+ nerv.printf("%s Displaying result:\n", global_conf.sche_log_pre)

+ nerv.printf("%s %s\n", global_conf.sche_log_pre, result:status("birnn"))

+ nerv.printf("%s Doing on %s end.\n", global_conf.sche_log_pre, fn)

+

+ return result

+end

+

+ nerv.printf("al_sen_start:%s\n", tostring(self.stat.al_sen_start))

+local LSTMLayerT = nerv.class('nerv.LSTMLayerTv2', 'nerv.LayerT')

+--a version of LSTM that only feed h into the gates

+

+function LSTMLayerT:__init(id, global_conf, layer_conf)

+ --input1:x input2:h input3:c

+ self.id = id

+ self.dim_in = layer_conf.dim_in

+ self.dim_out = layer_conf.dim_out

+ self.gconf = global_conf

+

+ --prepare a DAGLayerT to hold the lstm structure

+ local pr = layer_conf.pr

+ if pr == nil then

+ pr = nerv.ParamRepo()

+ end

+

+ local function ap(str)

+ return self.id .. '.' .. str

+ end

+

+ local layers = {

+ ["nerv.CombinerLayer"] = {

+ [ap("inputXDup")] = {{}, {["dim_in"] = {self.dim_in[1]},

+ ["dim_out"] = {self.dim_in[1], self.dim_in[1], self.dim_in[1], self.dim_in[1]}, ["lambda"] = {1}}},

+ [ap("inputHDup")] = {{}, {["dim_in"] = {self.dim_in[2]},

+ ["dim_out"] = {self.dim_in[2], self.dim_in[2], self.dim_in[2], self.dim_in[2]}, ["lambda"] = {1}}},

+ [ap("inputCDup")] = {{}, {["dim_in"] = {self.dim_in[3]},

+ ["dim_out"] = {self.dim_in[3]}, ["lambda"] = {1}}},

+ [ap("mainCDup")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]},

+ ["dim_out"] = {self.dim_in[3], self.dim_in[3]}, ["lambda"] = {1, 1}}},

+ },

+ ["nerv.AffineLayer"] = {

+ [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]},

+ ["dim_out"] = {self.dim_out[1]}, ["pr"] = pr}},

+ },

+ ["nerv.TanhLayer"] = {

+ [ap("mainTanhL")] = {{}, {["dim_in"] = {self.dim_out[1]}, ["dim_out"] = {self.dim_out[1]}}},

+ [ap("outputTanhL")] = {{}, {["dim_in"] = {self.dim_out[1]}, ["dim_out"] = {self.dim_out[1]}}},

+ },

+ ["nerv.GateFLayer"] = {

+ [ap("forgetGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]},

+ ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}},

+ [ap("inputGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]},

+ ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}},

+ [ap("outputGateL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]},

+ ["dim_out"] = {self.dim_in[3]}, ["pr"] = pr}},

+

+ },

+ ["nerv.ElemMulLayer"] = {

+ [ap("inputGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}},

+ [ap("forgetGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}},

+ [ap("outputGMulL")] = {{}, {["dim_in"] = {self.dim_in[3], self.dim_in[3]}, ["dim_out"] = {self.dim_in[3]}}},

+ },

+ }

+

+ local layerRepo = nerv.LayerRepo(layers, pr, global_conf)

+

+ local connections_t = {

+ ["<input>[1]"] = ap("inputXDup[1]"),

+ ["<input>[2]"] = ap("inputHDup[1]"),

+ ["<input>[3]"] = ap("inputCDup[1]"),

+

+ [ap("inputXDup[1]")] = ap("mainAffineL[1]"),

+ [ap("inputHDup[1]")] = ap("mainAffineL[2]"),

+

+ [ap("mainAffineL[1]")] = ap("mainTanhL[1]"),

+

+ [ap("inputXDup[2]")] = ap("inputGateL[1]"),

+ [ap("inputHDup[2]")] = ap("inputGateL[2]"),

+

+ [ap("inputXDup[3]")] = ap("forgetGateL[1]"),

+ [ap("inputHDup[3]")] = ap("forgetGateL[2]"),

+

+ [ap("mainTanhL[1]")] = ap("inputGMulL[1]"),

+ [ap("inputGateL[1]")] = ap("inputGMulL[2]"),

+

+ [ap("inputCDup[1]")] = ap("forgetGMulL[1]"),

+ [ap("forgetGateL[1]")] = ap("forgetGMulL[2]"),

+

+ [ap("inputGMulL[1]")] = ap("mainCDup[1]"),

+ [ap("forgetGMulL[1]")] = ap("mainCDup[2]"),

+

+ [ap("inputXDup[4]")] = ap("outputGateL[1]"),

+ [ap("inputHDup[4]")] = ap("outputGateL[2]"),

+

+ [ap("mainCDup[2]")] = "<output>[2]",

+ [ap("mainCDup[1]")] = ap("outputTanhL[1]"),

+

+ [ap("outputTanhL[1]")] = ap("outputGMulL[1]"),

+ [ap("outputGateL[1]")] = ap("outputGMulL[2]"),

+

+ [ap("outputGMulL[1]")] = "<output>[1]",

+ }

+ self.dagL = nerv.DAGLayerT(self.id, global_conf,

+ {["dim_in"] = self.dim_in, ["dim_out"] = self.dim_out, ["sub_layers"] = layerRepo,

+ ["connections"] = connections_t})

+

+ self:check_dim_len(3, 2) -- x, h, c and h, c

+end

+

+function LSTMLayerT:init(batch_size, chunk_size)

+ self.dagL:init(batch_size, chunk_size)

+end

+

+function LSTMLayerT:batch_resize(batch_size, chunk_size)

+ self.dagL:batch_resize(batch_size, chunk_size)

+end

+

+function LSTMLayerT:update(bp_err, input, output, t)

+ self.dagL:update(bp_err, input, output, t)

+end

+

+function LSTMLayerT:propagate(input, output, t)

+ self.dagL:propagate(input, output, t)

+end

+

+function LSTMLayerT:back_propagate(bp_err, next_bp_err, input, output, t)

+ self.dagL:back_propagate(bp_err, next_bp_err, input, output, t)

+end

+

+function LSTMLayerT:get_params()

+ return self.dagL:get_params()

+end

+root_dir = '/home/slhome/txh18/workspace'

+data_dir = root_dir .. '/ptb/DATA'

+qdata_dir = root_dir .. '/ptb/questionGen/gen'

+

+ lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5,

+ hidden_size = 300,

+ layer_num = 1,

+ max_iter = 35,

+ max_sen_len = 90,

+ work_dir_base = '/home/slhome/txh18/workspace/ptb/EXP-nerv/lstmlm_v1.0'

+elseif (set == "twitter") then

+

+root_dir = '/home/slhome/txh18/workspace'

+data_dir = root_dir .. '/twitter_new/DATA'

+train_fn = data_dir .. '/twitter.choose.adds'

+valid_fn = data_dir .. '/twitter.valid.adds'

+test_fn = data_dir .. '/comm.test.choose-ppl.adds'

+vocab_fn = data_dir .. '/twitter.choose.train.vocab'

+

+--qdata_dir = root_dir .. '/ptb/questionGen/gen'

+

+global_conf = {

+ lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5,

+ cumat_type = nerv.CuMatrixFloat,

+ mmat_type = nerv.MMatrixFloat,

+ nn_act_default = 0,

+

+ hidden_size = 300,

+ layer_num = 1,

+ chunk_size = 15,

+ batch_size = 20,

+ max_iter = 35,

+ lr_decay = 1.003,

+ decay_iter = 10,

+ param_random = function() return (math.random() / 5 - 0.1) end,

+ dropout_str = "0",

+

+ train_fn = train_fn,

+ valid_fn = valid_fn,

+ test_fn = test_fn,

+ vocab_fn = vocab_fn,

+ max_sen_len = 90,

+ sche_log_pre = "[SCHEDULER]:",

+ log_w_num = 40000, --give a message when log_w_num words have been processed

+ timer = nerv.Timer(),

+ work_dir_base = root_dir .. '/twitter_new/EXP-nerv/lstmlm_v1.0'

+}

+

+valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn'

+train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn'

+test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn'

+vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn'

+ max_sen_len = 80,

+lr_half = false --can not be local, to be set by loadstring

+start_iter = -1

+start_lr = global_conf.lrate

+ppl_last = 100000

+commands_str = "train:test"

+commands = {}

+test_iter = -1

+--for testout(question)

+q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds"

+global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str

+global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time())

+global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-')

+global_conf.lrate = start_lr

+

+nerv.printf("%s creating work_dir(%s)...\n", global_conf.sche_log_pre, global_conf.work_dir)

+nerv.LMUtil.wait(2)

+nerv.LMUtil.wait(2)

+nerv.printf("lr_half:\t%s\n", tostring(lr_half))

+nerv.printf("start_iter:\t%s\n", tostring(start_iter))

+nerv.printf("ppl_last:\t%s\n", tostring(ppl_last))

+nerv.printf("commands_str:\t%s\n", commands_str)

+nerv.printf("test_iter:\t%s\n", tostring(test_iter))

+

+if commands["testout"] == 1 then

+ nerv.printf("===TEST OUT===\n")

+ nerv.printf("q_file:\t%s\n", q_file)

+ local q_fn = q_file --qdata_dir .. '/' .. q_file

+ global_conf.sche_log_pre = "[SCHEDULER TESTOUT]:"

+ if final_iter ~= -1 and test_iter == -1 then

+ test_iter = final_iter

+ end

+ if test_iter == -1 then

+ test_iter = "final"

+ end

+ tnn = load_net(global_conf, test_iter)

+ global_conf.dropout_rate = 0

+ LMTrainer.lm_process_file_rnn(global_conf, q_fn, tnn, false,

+ {["one_sen_report"] = true}) --false update!

+end --if commands["testout"]

+

+

+</s> 你好 我 是 一个 人 </s>

+</s> 想 一起 玩 吗 </s>

+</s> 一个 人 很 好 玩 </s>

+</s> 不 想 一个 人 玩 </s>

+</s> 不 想 一个 人 玩 </s>

+local GateFFFLayer = nerv.class('nerv.GateFLayer', 'nerv.Layer') --Full matrix gate

+

+ for i = 1, #self.dim_in do

+ self["ltp" .. i] = self:find_param("ltp" .. i, layer_conf, global_conf, nerv.LinearTransParam, {self.dim_in[i], self.dim_out[1]}) --layer_conf.ltp

+ end

+ self:check_dim_len(-1, 1) --accept multiple inputs

+ for i = 1, #self.dim_in do

+ if self["ltp" .. i].trans:ncol() ~= self.bp.trans:ncol() then

+ nerv.error("mismatching dimensions of linear transform and bias paramter")

+ end

+ if self.dim_in[i] ~= self["ltp" .. i].trans:nrow() then

+ nerv.error("mismatching dimensions of linear transform parameter and input")

+ end

+ self["ltp"..i]:train_init()

+

+ for i = 2, #self.dim_in do

+ output[1]:mul(input[i], self["ltp" .. i].trans, 1.0, 1.0, 'N', 'N')

+ end

+ for i = 1, #self.dim_in do

+ next_bp_err[i]:mul(self.err_bakm, self["ltp" .. i].trans, 1.0, 0.0, 'N', 'T')

+ end

+ for i = 1, #self.dim_in do

+ self["ltp" .. i]:update_by_err_input(self.err_bakm, input[i])

+ end

+ local pr = nerv.ParamRepo({self.bp})

+ for i = 1, #self.dim_in do

+ pr:add(self["ltp" .. i].id, self["ltp" .. i])

+ end

+ return pr

+ fprintf(stdout, "reading chunk %d from %d\n", i, (int)offset);

+ nerv.info("initing DAGLayerT %s...", self.id)

+ ["dim_out"] = {self.dim_in[3], self.dim_in[3], self.dim_in[3]}, ["lambda"] = {1}}},

+ [ap("mainAffineL")] = {{}, {["dim_in"] = {self.dim_in[1], self.dim_in[2]},

+ ["nerv.GateFLayer"] = {

+ [ap("inputCDup[1]")] = ap("inputGateL[3]"),

+ [ap("inputCDup[2]")] = ap("forgetGateL[3]"),

+ [ap("inputCDup[3]")] = ap("forgetGMulL[1]"),

+ nerv.log_fh:flush()

+function TNN.make_initial_store(st, p, dim, batch_size, chunk_size, extend_t, global_conf, st_c, p_c, t_c)

+ --Return a table of matrix storage from time (1-extend_t)..(chunk_size+extend_t)

+ for i = 1 - extend_t - 1, chunk_size + extend_t + 1 do --intentionally allocated more time

+

+ self.extend_t = layer_conf.extend_t --TNN will allocate storage of time for 1-extend_t .. chunk_size+extend_t

+ if self.extend_t == nil then

+ self.extend_t = 5

+ end

+ nerv.info("tnn(%s) will extend storage beyond MB border for time steps %d...", id, self.extend_t)

+

+ nerv.info("TNN initing storage %s->%s", ref_from.layer.id, ref_to.layer.id)

+ self.make_initial_store(ref_from.outputs_m, port_from, dim, batch_size, chunk_size, self.extend_t, self.gconf, ref_to.inputs_m, port_to, time)

+ self.make_initial_store(ref_from.err_inputs_m, port_from, dim, batch_size, chunk_size, self.extend_t, self.gconf, ref_to.err_outputs_m, port_to, time)

+ self.make_initial_store(ref.outputs_m, p, self.dim_out[i], batch_size, chunk_size, self.extend_t, self.gconf, self.outputs_m, i, 0)

+ self.make_initial_store(ref.err_inputs_m, p, self.dim_out[i], batch_size, chunk_size, self.extend_t, self.gconf, self.err_inputs_m, i, 0)

+ self.make_initial_store(ref.inputs_m, p, self.dim_in[i], batch_size, chunk_size, self.extend_t, self.gconf, self.inputs_m, i, 0)

+ self.make_initial_store(ref.err_outputs_m, p, self.dim_in[i], batch_size, chunk_size, self.extend_t, self.gconf, self.err_outputs_m, i, 0)

+ for t = 1 - self.extend_t, self.chunk_size + self.extend_t do

+ for t = 1 - self.extend_t, self.chunk_size + self.extend_t do

+ for i = self.extend_t, 1, -1 do

+ list_t[i] = 1 - i

+ if t < 1 - self.extend_t or t > 0 then