From a1d8c0a2369ea72df77821f7b298903e9470e676 Mon Sep 17 00:00:00 2001 From: txh18 Date: Fri, 5 Feb 2016 21:19:58 +0800 Subject: ... --- nerv/examples/lmptb/m-tests/lm_sampler_test.lua | 454 ++++++++++++++++++++++++ nerv/examples/lmptb/sample_grulm_ptb_main.lua | 454 ------------------------ 2 files changed, 454 insertions(+), 454 deletions(-) create mode 100644 nerv/examples/lmptb/m-tests/lm_sampler_test.lua delete mode 100644 nerv/examples/lmptb/sample_grulm_ptb_main.lua diff --git a/nerv/examples/lmptb/m-tests/lm_sampler_test.lua b/nerv/examples/lmptb/m-tests/lm_sampler_test.lua new file mode 100644 index 0000000..42a5787 --- /dev/null +++ b/nerv/examples/lmptb/m-tests/lm_sampler_test.lua @@ -0,0 +1,454 @@ +require 'lmptb.lmvocab' +require 'lmptb.lmfeeder' +require 'lmptb.lmutil' +require 'lmptb.layer.init' +--require 'tnn.init' +require 'lmptb.lmseqreader' +require 'lm_trainer' +require 'lm_sampler' + +--[[global function rename]]-- +--local printf = nerv.printf +local LMTrainer = nerv.LMTrainer +--[[global function rename ends]]-- + +function prepare_parameters(global_conf, fn) + nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) + + global_conf.paramRepo = nerv.ParamRepo() + local paramRepo = global_conf.paramRepo + + nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, fn) + paramRepo:import({fn}, nil, global_conf) + + nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) + + return nil +end + +--global_conf: table +--Returns: nerv.LayerRepo +function prepare_layers(global_conf) + nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) + + local pr = global_conf.paramRepo + + local du = false + + --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}} + --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} + + local layers = { + ["nerv.GRULayerT"] = { + ["gruL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}}, + }, + + ["nerv.DropoutLayerT"] = { + ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, + }, + + ["nerv.SelectLinearLayer"] = { + ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, + }, + + ["nerv.CombinerLayer"] = { + ["combinerL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, + }, + + ["nerv.AffineLayer"] = { + ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, + }, + + ["nerv.SoftmaxCELayerT"] = { + ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, + }, + } + + for l = 2, global_conf.layer_num do + layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.GRULayerT"]["gruL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}} + layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} + end + --[[ --we do not need those in the new tnn framework + printf("%s adding %d bptt layers...\n", global_conf.sche_log_pre, global_conf.bptt) + for i = 1, global_conf.bptt do + layers["nerv.IndRecurrentLayer"]["recurrentL" .. (i + 1)] = recurrentLconfig + layers["nerv.SigmoidLayer"]["sigmoidL" .. (i + 1)] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} + layers["nerv.SelectLinearLayer"]["selectL" .. (i + 1)] = {{["ltp"] = "ltp_ih"}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}}} + end + --]] + + local layerRepo = nerv.LayerRepo(layers, pr, global_conf) + nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) + return layerRepo +end + +--global_conf: table +--layerRepo: nerv.LayerRepo +--Returns: a nerv.TNN +function prepare_tnn(global_conf, layerRepo) + nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) + + --input: input_w, input_w, ... input_w_now, last_activation + local connections_t = { + {"[1]", "selectL1[1]", 0}, + + --{"selectL1[1]", "recurrentL1[1]", 0}, + --{"recurrentL1[1]", "sigmoidL1[1]", 0}, + --{"sigmoidL1[1]", "combinerL1[1]", 0}, + --{"combinerL1[1]", "recurrentL1[2]", 1}, + + {"selectL1[1]", "gruL1[1]", 0}, + {"gruL1[1]", "combinerL1[1]", 0}, + {"combinerL1[1]", "gruL1[2]", 1}, + {"combinerL1[2]", "dropoutL1[1]", 0}, + + {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, + {"outputL[1]", "softmaxL[1]", 0}, + {"[2]", "softmaxL[2]", 0}, + {"softmaxL[1]", "[1]", 0} + } + + for l = 2, global_conf.layer_num do + table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "gruL"..l.."[1]", 0}) + table.insert(connections_t, {"gruL"..l.."[1]", "combinerL"..l.."[1]", 0}) + table.insert(connections_t, {"combinerL"..l.."[1]", "gruL"..l.."[2]", 1}) + table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) + end + + --[[ + printf("%s printing DAG connections:\n", global_conf.sche_log_pre) + for key, value in pairs(connections_t) do + printf("\t%s->%s\n", key, value) + end + ]]-- + + local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, + ["dim_out"] = {1}, ["sub_layers"] = layerRepo, + ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, + }) + + tnn:init(global_conf.batch_size, global_conf.chunk_size) + + nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) + return tnn +end + +function load_net_tnn(global_conf, fn) + prepare_parameters(global_conf, fn) + local layerRepo = prepare_layers(global_conf) + local tnn = prepare_tnn(global_conf, layerRepo) + return tnn +end + +function prepare_sampler(sm_conf) + sm_conf.pr = nerv.ParamRepo() + sm_conf.pr:import({sm_conf.fn_to_sample}, nil, sm_conf) + + local layers = { + ["nerv.GRULayerT"] = { + ["gruL1"] = {{}, {["dim_in"] = {sm_conf.hidden_size, sm_conf.hidden_size}, ["dim_out"] = {sm_conf.hidden_size}, ["pr"] = sm_conf.pr}}, + }, + ["nerv.DropoutLayerT"] = { + ["dropoutL1"] = {{}, {["dim_in"] = {sm_conf.hidden_size}, ["dim_out"] = {sm_conf.hidden_size}}}, + }, + ["nerv.SelectLinearLayer"] = { + ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {sm_conf.hidden_size}, ["vocab"] = sm_conf.vocab, ["pr"] = sm_conf.pr}}, + }, + ["nerv.CombinerLayer"] = { + ["combinerL1"] = {{}, {["dim_in"] = {sm_conf.hidden_size}, ["dim_out"] = {sm_conf.hidden_size, sm_conf.hidden_size}, ["lambda"] = {1}}}, + }, + ["nerv.AffineLayer"] = { + ["outputL"] = {{}, {["dim_in"] = {sm_conf.hidden_size}, ["dim_out"] = {sm_conf.vocab:size()}, ["pr"] = sm_conf.pr}}, + }, + ["nerv.SoftmaxCELayerT"] = { + ["softmaxL"] = {{}, {["dim_in"] = {sm_conf.vocab:size(), sm_conf.vocab:size()}, ["dim_out"] = {1}}}, + }, + } + local layerRepo = nerv.LayerRepo(layers, sm_conf.pr, sm_conf) + + local connections_t = { + ["[1]"] = "selectL1[1]", + + ["selectL1[1]"] = "gruL1[1]", + ["gruL1[1]"] = "combinerL1[1]", + ["[2]"] = "gruL1[2]", + --{"combinerL1[2]", "dropoutL1[1]", 0}, + + ["combinerL" .. global_conf.layer_num .. "[1]"] = "outputL[1]", + ["outputL[1]"] = "[1]", + ["combinerL1[2]"] = "[2]", + } + + if sm_conf.layer_num > 1 then + nerv.error("multiple layer is currently not supported(not hard to implement though)") + end + + local dagL = nerv.DAGLayerT("dagL", sm_conf, {["dim_in"] = {1, sm_conf.hidden_size}, + ["dim_out"] = {sm_conf.vocab:size(), sm_conf.hidden_size}, ["sub_layers"] = layerRepo, + ["connections"] = connections_t + }) + + local sampler = nerv.LMSampler(sm_conf) + sampler:load_dagL(dagL) + + return sampler +end + +local train_fn, valid_fn, test_fn +global_conf = {} +local set = arg[1] --"test" + +root_dir = '/home/slhome/txh18/workspace' + +if (set == "ptb") then + +data_dir = root_dir .. '/ptb/DATA' +train_fn = data_dir .. '/ptb.train.txt.adds' +valid_fn = data_dir .. '/ptb.valid.txt.adds' +test_fn = data_dir .. '/ptb.test.txt.adds' +vocab_fn = data_dir .. '/vocab' + +qdata_dir = root_dir .. '/ptb/questionGen/gen' + +global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 32, + max_iter = 35, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0.5", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 90, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/ptb/EXP-nerv/grulm_v1.0', + + fn_to_sample = root_dir .. '/ptb/EXP-nerv/grulm_v1.0h300l1ch15ba32slr0.15wc1e-05dr0.5/params.final', +} + +sm_conf = { + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + batch_size = 32, + chunk_size = 85, --largest sample sentence length + max_iter = 35, + max_sen_len = 90, + sche_log_pre = "[SAMPLER_S]:", + + timer = global_conf.timer, + fn_to_sample = root_dir .. '/ptb/EXP-nerv/grulm_v1.0h300l1ch15ba32slr0.15wc1e-05dr0.5/params.final', +} + +elseif (set == "msr_sc") then + +data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' +train_fn = data_dir .. '/normed_all.sf.len60.adds.train' +valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' +test_fn = data_dir .. '/answer_normed.adds' +vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' + +global_conf = { + lrate = 1, wcost = 1e-6, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 10, + max_iter = 30, + decay_iter = 10, + lr_decay = 1.003, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 400000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' +} + +elseif (set == "twitter") then + + data_dir = root_dir .. '/twitter_new/DATA' + train_fn = data_dir .. '/twitter.choose2.adds' + valid_fn = data_dir .. '/twitter.valid.adds' + test_fn = data_dir .. '/comm.test.choose-ppl.adds' + vocab_fn = data_dir .. '/twitter.choose.train.vocab' + --qdata_dir = root_dir .. '/ptb/questionGen/gen' + global_conf = { + lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 300, + layer_num = 1, + chunk_size = 15, + batch_size = 32, + max_iter = 30, + lr_decay = 1.003, + decay_iter = 10, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0.5", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + vocab_fn = vocab_fn, + max_sen_len = 32, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 40000, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = root_dir .. '/twitter_new/EXP-nerv/grulm_v1.0' + } + +else + +valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' +vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' + +global_conf = { + lrate = 0.01, wcost = 1e-5, momentum = 0, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + nn_act_default = 0, + + hidden_size = 20, + layer_num = 1, + chunk_size = 2, + batch_size = 10, + max_iter = 3, + param_random = function() return (math.random() / 5 - 0.1) end, + dropout_str = "0", + + train_fn = train_fn, + valid_fn = valid_fn, + test_fn = test_fn, + max_sen_len = 80, + lr_decay = 1.003, + decay_iter = 10, + vocab_fn = vocab_fn, + sche_log_pre = "[SCHEDULER]:", + log_w_num = 10, --give a message when log_w_num words have been processed + timer = nerv.Timer(), + work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_lstmlm_test' +} + +end + +lr_half = false --can not be local, to be set by loadstring +start_iter = -1 +start_lr = nil +ppl_last = 100000 +commands_str = "sampling" --"train:test" +commands = {} +test_iter = -1 +--for testout(question) +q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds" + +if arg[2] ~= nil then + nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) + loadstring(arg[2])() + nerv.LMUtil.wait(0.5) +else + nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) +end + +global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str +global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' +global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' +global_conf.param_fn = global_conf.work_dir .. "/params" +global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) +global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) +global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') +commands = nerv.SUtil.parse_commands_set(commands_str) + +if start_lr ~= nil then + global_conf.lrate = start_lr +end + +--[[ +--redirecting log outputs! +nerv.SUtil.log_redirect(global_conf.log_fn) +nerv.LMUtil.wait(2) +]]-- + +----------------printing options--------------------------------- +nerv.printf("%s printing sm_conf...\n", sm_conf.sche_log_pre) +for id, value in pairs(sm_conf) do + nerv.printf("%s:\t%s\n", id, tostring(value)) +end +nerv.LMUtil.wait(2) + +nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) +nerv.printf("lr_half:\t%s\n", tostring(lr_half)) +nerv.printf("start_iter:\t%s\n", tostring(start_iter)) +nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) +nerv.printf("commands_str:\t%s\n", commands_str) +nerv.printf("test_iter:\t%s\n", tostring(test_iter)) +nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) +nerv.LMUtil.wait(2) +------------------printing options end------------------------------ + +math.randomseed(1) + +local vocab = nerv.LMVocab() +global_conf["vocab"] = vocab +sm_conf["vocab"] = global_conf.vocab +nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) +global_conf.vocab:build_file(global_conf.vocab_fn, false) +ppl_rec = {} + +local final_iter = -1 +if commands["test"] == 1 then + nerv.printf("===FINAL TEST===\n") + global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" + local tnn = load_net_tnn(global_conf, global_conf.fn_to_sample) + global_conf.dropout_rate = 0 + LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["test"] + +if commands["sampling"] == 1 then + nerv.printf("===SAMPLE===\n") + global_conf.sche_log_pre = "[SCHEDULER SAMPLING]:" + local sampler = prepare_sampler(sm_conf) + for k = 1, 1 do + local res = sampler:lm_sample_rnn_dagL(10, {}) + for i = 1, #res do + for j = 1, #res[i] do + nerv.printf("%s(%f) ", res[i][j].w, res[i][j].p) + end + nerv.printf("\n") + end + end + --global_conf.dropout_rate = 0 + --LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! +end --if commands["sampling"] + + diff --git a/nerv/examples/lmptb/sample_grulm_ptb_main.lua b/nerv/examples/lmptb/sample_grulm_ptb_main.lua deleted file mode 100644 index 42a5787..0000000 --- a/nerv/examples/lmptb/sample_grulm_ptb_main.lua +++ /dev/null @@ -1,454 +0,0 @@ -require 'lmptb.lmvocab' -require 'lmptb.lmfeeder' -require 'lmptb.lmutil' -require 'lmptb.layer.init' ---require 'tnn.init' -require 'lmptb.lmseqreader' -require 'lm_trainer' -require 'lm_sampler' - ---[[global function rename]]-- ---local printf = nerv.printf -local LMTrainer = nerv.LMTrainer ---[[global function rename ends]]-- - -function prepare_parameters(global_conf, fn) - nerv.printf("%s preparing parameters...\n", global_conf.sche_log_pre) - - global_conf.paramRepo = nerv.ParamRepo() - local paramRepo = global_conf.paramRepo - - nerv.printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, fn) - paramRepo:import({fn}, nil, global_conf) - - nerv.printf("%s preparing parameters end.\n", global_conf.sche_log_pre) - - return nil -end - ---global_conf: table ---Returns: nerv.LayerRepo -function prepare_layers(global_conf) - nerv.printf("%s preparing layers...\n", global_conf.sche_log_pre) - - local pr = global_conf.paramRepo - - local du = false - - --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}} - --local recurrentLconfig = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du, ["pr"] = pr}} - - local layers = { - ["nerv.GRULayerT"] = { - ["gruL1"] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}}, - }, - - ["nerv.DropoutLayerT"] = { - ["dropoutL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}, - }, - - ["nerv.SelectLinearLayer"] = { - ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}, ["vocab"] = global_conf.vocab, ["pr"] = pr}}, - }, - - ["nerv.CombinerLayer"] = { - ["combinerL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}, - }, - - ["nerv.AffineLayer"] = { - ["outputL"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du, ["pr"] = pr}}, - }, - - ["nerv.SoftmaxCELayerT"] = { - ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}}, - }, - } - - for l = 2, global_conf.layer_num do - layers["nerv.DropoutLayerT"]["dropoutL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} - layers["nerv.GRULayerT"]["gruL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["pr"] = pr}} - layers["nerv.CombinerLayer"]["combinerL" .. l] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}} - end - --[[ --we do not need those in the new tnn framework - printf("%s adding %d bptt layers...\n", global_conf.sche_log_pre, global_conf.bptt) - for i = 1, global_conf.bptt do - layers["nerv.IndRecurrentLayer"]["recurrentL" .. (i + 1)] = recurrentLconfig - layers["nerv.SigmoidLayer"]["sigmoidL" .. (i + 1)] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}} - layers["nerv.SelectLinearLayer"]["selectL" .. (i + 1)] = {{["ltp"] = "ltp_ih"}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}}} - end - --]] - - local layerRepo = nerv.LayerRepo(layers, pr, global_conf) - nerv.printf("%s preparing layers end.\n", global_conf.sche_log_pre) - return layerRepo -end - ---global_conf: table ---layerRepo: nerv.LayerRepo ---Returns: a nerv.TNN -function prepare_tnn(global_conf, layerRepo) - nerv.printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre) - - --input: input_w, input_w, ... input_w_now, last_activation - local connections_t = { - {"[1]", "selectL1[1]", 0}, - - --{"selectL1[1]", "recurrentL1[1]", 0}, - --{"recurrentL1[1]", "sigmoidL1[1]", 0}, - --{"sigmoidL1[1]", "combinerL1[1]", 0}, - --{"combinerL1[1]", "recurrentL1[2]", 1}, - - {"selectL1[1]", "gruL1[1]", 0}, - {"gruL1[1]", "combinerL1[1]", 0}, - {"combinerL1[1]", "gruL1[2]", 1}, - {"combinerL1[2]", "dropoutL1[1]", 0}, - - {"dropoutL"..global_conf.layer_num.."[1]", "outputL[1]", 0}, - {"outputL[1]", "softmaxL[1]", 0}, - {"[2]", "softmaxL[2]", 0}, - {"softmaxL[1]", "[1]", 0} - } - - for l = 2, global_conf.layer_num do - table.insert(connections_t, {"dropoutL"..(l-1).."[1]", "gruL"..l.."[1]", 0}) - table.insert(connections_t, {"gruL"..l.."[1]", "combinerL"..l.."[1]", 0}) - table.insert(connections_t, {"combinerL"..l.."[1]", "gruL"..l.."[2]", 1}) - table.insert(connections_t, {"combinerL"..l.."[2]", "dropoutL"..l.."[1]", 0}) - end - - --[[ - printf("%s printing DAG connections:\n", global_conf.sche_log_pre) - for key, value in pairs(connections_t) do - printf("\t%s->%s\n", key, value) - end - ]]-- - - local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, - ["dim_out"] = {1}, ["sub_layers"] = layerRepo, - ["connections"] = connections_t, ["clip_t"] = global_conf.clip_t, - }) - - tnn:init(global_conf.batch_size, global_conf.chunk_size) - - nerv.printf("%s Initing TNN end.\n", global_conf.sche_log_pre) - return tnn -end - -function load_net_tnn(global_conf, fn) - prepare_parameters(global_conf, fn) - local layerRepo = prepare_layers(global_conf) - local tnn = prepare_tnn(global_conf, layerRepo) - return tnn -end - -function prepare_sampler(sm_conf) - sm_conf.pr = nerv.ParamRepo() - sm_conf.pr:import({sm_conf.fn_to_sample}, nil, sm_conf) - - local layers = { - ["nerv.GRULayerT"] = { - ["gruL1"] = {{}, {["dim_in"] = {sm_conf.hidden_size, sm_conf.hidden_size}, ["dim_out"] = {sm_conf.hidden_size}, ["pr"] = sm_conf.pr}}, - }, - ["nerv.DropoutLayerT"] = { - ["dropoutL1"] = {{}, {["dim_in"] = {sm_conf.hidden_size}, ["dim_out"] = {sm_conf.hidden_size}}}, - }, - ["nerv.SelectLinearLayer"] = { - ["selectL1"] = {{}, {["dim_in"] = {1}, ["dim_out"] = {sm_conf.hidden_size}, ["vocab"] = sm_conf.vocab, ["pr"] = sm_conf.pr}}, - }, - ["nerv.CombinerLayer"] = { - ["combinerL1"] = {{}, {["dim_in"] = {sm_conf.hidden_size}, ["dim_out"] = {sm_conf.hidden_size, sm_conf.hidden_size}, ["lambda"] = {1}}}, - }, - ["nerv.AffineLayer"] = { - ["outputL"] = {{}, {["dim_in"] = {sm_conf.hidden_size}, ["dim_out"] = {sm_conf.vocab:size()}, ["pr"] = sm_conf.pr}}, - }, - ["nerv.SoftmaxCELayerT"] = { - ["softmaxL"] = {{}, {["dim_in"] = {sm_conf.vocab:size(), sm_conf.vocab:size()}, ["dim_out"] = {1}}}, - }, - } - local layerRepo = nerv.LayerRepo(layers, sm_conf.pr, sm_conf) - - local connections_t = { - ["[1]"] = "selectL1[1]", - - ["selectL1[1]"] = "gruL1[1]", - ["gruL1[1]"] = "combinerL1[1]", - ["[2]"] = "gruL1[2]", - --{"combinerL1[2]", "dropoutL1[1]", 0}, - - ["combinerL" .. global_conf.layer_num .. "[1]"] = "outputL[1]", - ["outputL[1]"] = "[1]", - ["combinerL1[2]"] = "[2]", - } - - if sm_conf.layer_num > 1 then - nerv.error("multiple layer is currently not supported(not hard to implement though)") - end - - local dagL = nerv.DAGLayerT("dagL", sm_conf, {["dim_in"] = {1, sm_conf.hidden_size}, - ["dim_out"] = {sm_conf.vocab:size(), sm_conf.hidden_size}, ["sub_layers"] = layerRepo, - ["connections"] = connections_t - }) - - local sampler = nerv.LMSampler(sm_conf) - sampler:load_dagL(dagL) - - return sampler -end - -local train_fn, valid_fn, test_fn -global_conf = {} -local set = arg[1] --"test" - -root_dir = '/home/slhome/txh18/workspace' - -if (set == "ptb") then - -data_dir = root_dir .. '/ptb/DATA' -train_fn = data_dir .. '/ptb.train.txt.adds' -valid_fn = data_dir .. '/ptb.valid.txt.adds' -test_fn = data_dir .. '/ptb.test.txt.adds' -vocab_fn = data_dir .. '/vocab' - -qdata_dir = root_dir .. '/ptb/questionGen/gen' - -global_conf = { - lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 300, - layer_num = 1, - chunk_size = 15, - batch_size = 32, - max_iter = 35, - lr_decay = 1.003, - decay_iter = 10, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0.5", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - vocab_fn = vocab_fn, - max_sen_len = 90, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 40000, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = root_dir .. '/ptb/EXP-nerv/grulm_v1.0', - - fn_to_sample = root_dir .. '/ptb/EXP-nerv/grulm_v1.0h300l1ch15ba32slr0.15wc1e-05dr0.5/params.final', -} - -sm_conf = { - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 300, - layer_num = 1, - batch_size = 32, - chunk_size = 85, --largest sample sentence length - max_iter = 35, - max_sen_len = 90, - sche_log_pre = "[SAMPLER_S]:", - - timer = global_conf.timer, - fn_to_sample = root_dir .. '/ptb/EXP-nerv/grulm_v1.0h300l1ch15ba32slr0.15wc1e-05dr0.5/params.final', -} - -elseif (set == "msr_sc") then - -data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2' -train_fn = data_dir .. '/normed_all.sf.len60.adds.train' -valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev' -test_fn = data_dir .. '/answer_normed.adds' -vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab' - -global_conf = { - lrate = 1, wcost = 1e-6, momentum = 0, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 300, - layer_num = 1, - chunk_size = 15, - batch_size = 10, - max_iter = 30, - decay_iter = 10, - lr_decay = 1.003, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - vocab_fn = vocab_fn, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 400000, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test' -} - -elseif (set == "twitter") then - - data_dir = root_dir .. '/twitter_new/DATA' - train_fn = data_dir .. '/twitter.choose2.adds' - valid_fn = data_dir .. '/twitter.valid.adds' - test_fn = data_dir .. '/comm.test.choose-ppl.adds' - vocab_fn = data_dir .. '/twitter.choose.train.vocab' - --qdata_dir = root_dir .. '/ptb/questionGen/gen' - global_conf = { - lrate = 0.15, wcost = 1e-5, momentum = 0, clip_t = 5, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 300, - layer_num = 1, - chunk_size = 15, - batch_size = 32, - max_iter = 30, - lr_decay = 1.003, - decay_iter = 10, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0.5", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - vocab_fn = vocab_fn, - max_sen_len = 32, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 40000, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = root_dir .. '/twitter_new/EXP-nerv/grulm_v1.0' - } - -else - -valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' -train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' -test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' -vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text-chn' - -global_conf = { - lrate = 0.01, wcost = 1e-5, momentum = 0, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - nn_act_default = 0, - - hidden_size = 20, - layer_num = 1, - chunk_size = 2, - batch_size = 10, - max_iter = 3, - param_random = function() return (math.random() / 5 - 0.1) end, - dropout_str = "0", - - train_fn = train_fn, - valid_fn = valid_fn, - test_fn = test_fn, - max_sen_len = 80, - lr_decay = 1.003, - decay_iter = 10, - vocab_fn = vocab_fn, - sche_log_pre = "[SCHEDULER]:", - log_w_num = 10, --give a message when log_w_num words have been processed - timer = nerv.Timer(), - work_dir_base = '/home/slhome/txh18/workspace/nerv/play/testEXP/tnn_lstmlm_test' -} - -end - -lr_half = false --can not be local, to be set by loadstring -start_iter = -1 -start_lr = nil -ppl_last = 100000 -commands_str = "sampling" --"train:test" -commands = {} -test_iter = -1 ---for testout(question) -q_file = "/home/slhome/txh18/workspace/ptb/questionGen/gen/ptb.test.txt.q10rs1_Msss.adds" - -if arg[2] ~= nil then - nerv.printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2]) - loadstring(arg[2])() - nerv.LMUtil.wait(0.5) -else - nerv.printf("%s no user setting, all default...\n", global_conf.sche_log_pre) -end - -global_conf.work_dir = global_conf.work_dir_base .. 'h' .. global_conf.hidden_size .. 'l' .. global_conf.layer_num .. 'ch' .. global_conf.chunk_size .. 'ba' .. global_conf.batch_size .. 'slr' .. global_conf.lrate .. 'wc' .. global_conf.wcost .. 'dr' .. global_conf.dropout_str -global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf' -global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak' -global_conf.param_fn = global_conf.work_dir .. "/params" -global_conf.dropout_list = nerv.SUtil.parse_schedule(global_conf.dropout_str) -global_conf.log_fn = global_conf.work_dir .. '/log_lstm_tnn_' .. commands_str ..os.date("_TT%m_%d_%X",os.time()) -global_conf.log_fn, _ = string.gsub(global_conf.log_fn, ':', '-') -commands = nerv.SUtil.parse_commands_set(commands_str) - -if start_lr ~= nil then - global_conf.lrate = start_lr -end - ---[[ ---redirecting log outputs! -nerv.SUtil.log_redirect(global_conf.log_fn) -nerv.LMUtil.wait(2) -]]-- - -----------------printing options--------------------------------- -nerv.printf("%s printing sm_conf...\n", sm_conf.sche_log_pre) -for id, value in pairs(sm_conf) do - nerv.printf("%s:\t%s\n", id, tostring(value)) -end -nerv.LMUtil.wait(2) - -nerv.printf("%s printing training scheduling options...\n", global_conf.sche_log_pre) -nerv.printf("lr_half:\t%s\n", tostring(lr_half)) -nerv.printf("start_iter:\t%s\n", tostring(start_iter)) -nerv.printf("ppl_last:\t%s\n", tostring(ppl_last)) -nerv.printf("commands_str:\t%s\n", commands_str) -nerv.printf("test_iter:\t%s\n", tostring(test_iter)) -nerv.printf("%s printing training scheduling end.\n", global_conf.sche_log_pre) -nerv.LMUtil.wait(2) -------------------printing options end------------------------------ - -math.randomseed(1) - -local vocab = nerv.LMVocab() -global_conf["vocab"] = vocab -sm_conf["vocab"] = global_conf.vocab -nerv.printf("%s building vocab...\n", global_conf.sche_log_pre) -global_conf.vocab:build_file(global_conf.vocab_fn, false) -ppl_rec = {} - -local final_iter = -1 -if commands["test"] == 1 then - nerv.printf("===FINAL TEST===\n") - global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" - local tnn = load_net_tnn(global_conf, global_conf.fn_to_sample) - global_conf.dropout_rate = 0 - LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! -end --if commands["test"] - -if commands["sampling"] == 1 then - nerv.printf("===SAMPLE===\n") - global_conf.sche_log_pre = "[SCHEDULER SAMPLING]:" - local sampler = prepare_sampler(sm_conf) - for k = 1, 1 do - local res = sampler:lm_sample_rnn_dagL(10, {}) - for i = 1, #res do - for j = 1, #res[i] do - nerv.printf("%s(%f) ", res[i][j].w, res[i][j].p) - end - nerv.printf("\n") - end - end - --global_conf.dropout_rate = 0 - --LMTrainer.lm_process_file_rnn(global_conf, global_conf.test_fn, tnn, false) --false update! -end --if commands["sampling"] - - -- cgit v1.2.3