require 'lmptb.lmvocab'
require 'lmptb.lmfeeder'
require 'lmptb.lmutil'
require 'lmptb.layer.init'
require 'rnn.init'
require 'lmptb.lmseqreader'
require 'lm_trainer'

--[[global function rename]]--
local printf = nerv.printf
local LMTrainer = nerv.LMTrainer
--[[global function rename ends]]--

--global_conf: table
--first_time: bool
--Returns: a ParamRepo
function prepare_parameters(global_conf, iter)
    printf("%s preparing parameters...\n", global_conf.sche_log_pre) 
    
    if iter == -1 then --first time
        printf("%s first time, generating parameters...\n", global_conf.sche_log_pre) 
        ltp_ih = nerv.LinearTransParam("ltp_ih", global_conf)  
        ltp_ih.trans = global_conf.cumat_type(global_conf.vocab:size(), global_conf.hidden_size) --index 0 is for zero, others correspond to vocab index(starting from 1)
        ltp_ih.trans:generate(global_conf.param_random)

        ltp_hh = nerv.LinearTransParam("ltp_hh", global_conf)
        ltp_hh.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.hidden_size)
        ltp_hh.trans:generate(global_conf.param_random) 

        ltp_ho = nerv.LinearTransParam("ltp_ho", global_conf)
        ltp_ho.trans = global_conf.cumat_type(global_conf.hidden_size, global_conf.vocab:size())
        ltp_ho.trans:generate(global_conf.param_random)

        bp_h = nerv.BiasParam("bp_h", global_conf)
        bp_h.trans = global_conf.cumat_type(1, global_conf.hidden_size)
        bp_h.trans:generate(global_conf.param_random)

        bp_o = nerv.BiasParam("bp_o", global_conf)
        bp_o.trans = global_conf.cumat_type(1, global_conf.vocab:size())
        bp_o.trans:generate(global_conf.param_random)

        local f = nerv.ChunkFile(global_conf.param_fn .. '.0', 'w')
        f:write_chunk(ltp_ih)
        f:write_chunk(ltp_hh)
        f:write_chunk(ltp_ho)
        f:write_chunk(bp_h)
        f:write_chunk(bp_o)
        f:close()

        return nil
    end
    
    printf("%s loading parameter from file %s...\n", global_conf.sche_log_pre, global_conf.param_fn .. '.' .. tostring(iter)) 
    local paramRepo = nerv.ParamRepo()
    paramRepo:import({global_conf.param_fn .. '.' .. tostring(iter)}, nil, global_conf)

    printf("%s preparing parameters end.\n", global_conf.sche_log_pre)

    return paramRepo
end

--global_conf: table
--Returns: nerv.LayerRepo
function prepare_layers(global_conf, paramRepo)
    printf("%s preparing layers...\n", global_conf.sche_log_pre)
    
    local du = false

    --local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["break_id"] = global_conf.vocab:get_sen_entry().id, ["independent"] = global_conf.independent, ["clip"] = 10}}
    local recurrentLconfig = {{["bp"] = "bp_h", ["ltp_hh"] = "ltp_hh"}, {["dim_in"] = {global_conf.hidden_size, global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}, ["clip"] = 10, ["direct_update"] = du}}

    local layers = {
        ["nerv.AffineRecurrentLayer"] = {
            ["recurrentL1"] = recurrentLconfig, 
        },

        ["nerv.SelectLinearLayer"] = {
            ["selectL1"] = {{["ltp"] = "ltp_ih"}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}}},
        },
        
        ["nerv.SigmoidLayer"] = {
            ["sigmoidL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}
        },
        
        ["nerv.CombinerLayer"] = {
            ["combinerL1"] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size, global_conf.hidden_size}, ["lambda"] = {1}}}
        },

        ["nerv.AffineLayer"] = {
            ["outputL"] = {{["ltp"] = "ltp_ho", ["bp"] = "bp_o"}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.vocab:size()}, ["direct_update"] = du}},
        },

        ["nerv.SoftmaxCELayerT"] = {
            ["softmaxL"] = {{}, {["dim_in"] = {global_conf.vocab:size(), global_conf.vocab:size()}, ["dim_out"] = {1}}},
        },
    }
    
    --[[ --we do not need those in the new rnn framework
    printf("%s adding %d bptt layers...\n", global_conf.sche_log_pre, global_conf.bptt)
    for i = 1, global_conf.bptt do
        layers["nerv.IndRecurrentLayer"]["recurrentL" .. (i + 1)] = recurrentLconfig 
        layers["nerv.SigmoidLayer"]["sigmoidL" .. (i + 1)] = {{}, {["dim_in"] = {global_conf.hidden_size}, ["dim_out"] = {global_conf.hidden_size}}}
        layers["nerv.SelectLinearLayer"]["selectL" .. (i + 1)] = {{["ltp"] = "ltp_ih"}, {["dim_in"] = {1}, ["dim_out"] = {global_conf.hidden_size}}}
    end
    --]]

    local layerRepo = nerv.LayerRepo(layers, paramRepo, global_conf)
    printf("%s preparing layers end.\n", global_conf.sche_log_pre)
    return layerRepo
end

--global_conf: table
--layerRepo: nerv.LayerRepo
--Returns: a nerv.TNN
function prepare_tnn(global_conf, layerRepo)
    printf("%s Generate and initing TNN ...\n", global_conf.sche_log_pre)

    --input: input_w, input_w, ... input_w_now, last_activation
    local connections_t = {
        {"<input>[1]", "selectL1[1]", 0},
        {"selectL1[1]", "recurrentL1[1]", 0},  
        {"recurrentL1[1]", "sigmoidL1[1]", 0},
        {"sigmoidL1[1]", "combinerL1[1]", 0},
        {"combinerL1[1]", "recurrentL1[2]", 1},
        {"combinerL1[2]", "outputL[1]", 0},
        {"outputL[1]", "softmaxL[1]", 0},
        {"<input>[2]", "softmaxL[2]", 0},
        {"softmaxL[1]", "<output>[1]", 0}
    }

    --[[
    printf("%s printing DAG connections:\n", global_conf.sche_log_pre)
    for key, value in pairs(connections_t) do
        printf("\t%s->%s\n", key, value)
    end
    ]]--

    local tnn = nerv.TNN("TNN", global_conf, {["dim_in"] = {1, global_conf.vocab:size()}, ["dim_out"] = {1}, ["sub_layers"] = layerRepo,
            ["connections"] = connections_t, 
        })

    tnn:init(global_conf.batch_size, global_conf.chunk_size)

    printf("%s Initing TNN end.\n", global_conf.sche_log_pre)
    return tnn
end

function load_net(global_conf, next_iter)
    local paramRepo = prepare_parameters(global_conf, next_iter)
    local layerRepo = prepare_layers(global_conf, paramRepo)
    local tnn = prepare_tnn(global_conf, layerRepo)
    return tnn, paramRepo
end

local train_fn, valid_fn, test_fn
global_conf = {}
local set = arg[1] --"test"

if (set == "ptb") then

data_dir = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/PTBdata'
train_fn = data_dir .. '/ptb.train.txt.adds'
valid_fn = data_dir .. '/ptb.valid.txt.adds'
test_fn = data_dir .. '/ptb.test.txt.adds'
vocab_fn = data_dir .. '/vocab'

global_conf = {
    lrate = 1, wcost = 1e-5, momentum = 0,
    cumat_type = nerv.CuMatrixFloat,
    mmat_type = nerv.MMatrixFloat,
    nn_act_default = 0, 

    hidden_size = 400, --set to 400 for a stable good test PPL
    chunk_size = 15,
    batch_size = 10, 
    max_iter = 35,
    decay_iter = 16,
    param_random = function() return (math.random() / 5 - 0.1) end,

    train_fn = train_fn,
    valid_fn = valid_fn,
    test_fn = test_fn,
    vocab_fn = vocab_fn,
    sche_log_pre = "[SCHEDULER]:",
    log_w_num = 40000, --give a message when log_w_num words have been processed
    timer = nerv.Timer(),
    work_dir = '/home/slhome/txh18/workspace/nerv/play/dagL_test'
}

elseif (set == "msr_sc") then

data_dir = '/home/slhome/txh18/workspace/sentenceCompletion/DATA_PV2'
train_fn = data_dir .. '/normed_all.sf.len60.adds.train'
valid_fn = data_dir .. '/normed_all.sf.len60.adds.dev'
test_fn = data_dir .. '/answer_normed.adds'
vocab_fn = data_dir .. '/normed_all.choose.vocab30000.addqvocab'

global_conf = {
    lrate = 1, wcost = 1e-6, momentum = 0,
    cumat_type = nerv.CuMatrixFloat,
    mmat_type = nerv.MMatrixFloat,
    nn_act_default = 0, 

    hidden_size = 300,
    chunk_size = 15,
    batch_size = 10, 
    max_iter = 30,
    decay_iter = 10,
    param_random = function() return (math.random() / 5 - 0.1) end,

    train_fn = train_fn,
    valid_fn = valid_fn,
    test_fn = test_fn,
    vocab_fn = vocab_fn,
    sche_log_pre = "[SCHEDULER]:",
    log_w_num = 40000, --give a message when log_w_num words have been processed
    timer = nerv.Timer(),
    work_dir = '/home/slhome/txh18/workspace/sentenceCompletion/EXP-Nerv/rnnlm_test'
}

else

valid_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'
train_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'
test_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'
vocab_fn = '/home/slhome/txh18/workspace/nerv/nerv/nerv/examples/lmptb/m-tests/some-text'

global_conf = {
    lrate = 1, wcost = 1e-5, momentum = 0,
    cumat_type = nerv.CuMatrixFloat,
    mmat_type = nerv.CuMatrixFloat,
    nn_act_default = 0, 

    hidden_size = 20,
    chunk_size = 2,
    batch_size = 3, 
    max_iter = 3,
    param_random = function() return (math.random() / 5 - 0.1) end,

    train_fn = train_fn,
    valid_fn = valid_fn,
    test_fn = test_fn,
    vocab_fn = vocab_fn, 
    sche_log_pre = "[SCHEDULER]:",
    log_w_num = 10, --give a message when log_w_num words have been processed
    timer = nerv.Timer(),
    work_dir = '/home/slhome/txh18/workspace/nerv/play/dagL_test'
}

end

global_conf.train_fn_shuf = global_conf.work_dir .. '/train_fn_shuf'
global_conf.train_fn_shuf_bak = global_conf.train_fn_shuf .. '_bak'
global_conf.param_fn = global_conf.work_dir .. "/params"

lr_half = false --can not be local, to be set by loadstring
start_iter = -1
ppl_last = 100000
if (arg[2] ~= nil) then
    printf("%s applying arg[2](%s)...\n", global_conf.sche_log_pre, arg[2])
    loadstring(arg[2])() 
    nerv.LMUtil.wait(0.5)
else
    printf("%s not user setting, all default...\n", global_conf.sche_log_pre)
end

----------------printing options---------------------------------
printf("%s printing global_conf...\n", global_conf.sche_log_pre)
for id, value in pairs(global_conf) do
    print(id, value)
end
nerv.LMUtil.wait(2)
printf("%s printing training scheduling options...\n", global_conf.sche_log_pre)
print("lr_half", lr_half)
print("start_iter", start_iter)
print("ppl_last", ppl_last)
printf("%s printing training scheduling end.\n", global_conf.sche_log_pre)
nerv.LMUtil.wait(2)
------------------printing options end------------------------------

math.randomseed(1)

printf("%s creating work_dir...\n", global_conf.sche_log_pre)
os.execute("mkdir -p "..global_conf.work_dir)
os.execute("cp " .. global_conf.train_fn .. " " .. global_conf.train_fn_shuf)

local vocab = nerv.LMVocab()
global_conf["vocab"] = vocab
printf("%s building vocab...\n", global_conf.sche_log_pre)
global_conf.vocab:build_file(global_conf.vocab_fn, false)
ppl_rec = {} 

if start_iter == -1 then 
    prepare_parameters(global_conf, -1) --randomly generate parameters
end

if start_iter == -1 or start_iter == 0 then
    print("===INITIAL VALIDATION===") 
    local tnn, paramRepo = load_net(global_conf, 0)
    local result = LMTrainer.lm_process_file(global_conf, global_conf.valid_fn, tnn, false) --false update!
    nerv.LMUtil.wait(1)
    ppl_rec[0] = {} 
    ppl_rec[0].valid = result:ppl_all("rnn")  
    ppl_last = ppl_rec[0].valid 
    ppl_rec[0].train = 0 
    ppl_rec[0].test = 0
    ppl_rec[0].lr = 0 

    start_iter = 1

    print() 
end

local final_iter
for iter = start_iter, global_conf.max_iter, 1 do
    final_iter = iter --for final testing
    global_conf.sche_log_pre = "[SCHEDULER ITER"..iter.." LR"..global_conf.lrate.."]:" 
    tnn, paramRepo = load_net(global_conf, iter - 1) 
    printf("===ITERATION %d LR %f===\n", iter, global_conf.lrate) 
    result = LMTrainer.lm_process_file(global_conf, global_conf.train_fn_shuf, tnn, true) --true update!
    ppl_rec[iter] = {}
    ppl_rec[iter].train = result:ppl_all("rnn")
    --shuffling training file
    printf("%s shuffling training file\n", global_conf.sche_log_pre)
    os.execute('cp ' .. global_conf.train_fn_shuf .. ' ' .. global_conf.train_fn_shuf_bak)
    os.execute('cat ' .. global_conf.train_fn_shuf_bak .. ' | sort -R --random-source=/dev/zero > ' .. global_conf.train_fn_shuf)
    printf("===PEEK ON TEST %d===\n", iter) 
    result = LMTrainer.lm_process_file(global_conf, global_conf.test_fn, tnn, false) --false update!
    ppl_rec[iter].test = result:ppl_all("rnn")  
    printf("===VALIDATION %d===\n", iter) 
    result = LMTrainer.lm_process_file(global_conf, global_conf.valid_fn, tnn, false) --false update!
    ppl_rec[iter].valid = result:ppl_all("rnn") 
    ppl_rec[iter].lr = global_conf.lrate 
    if ((ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true) and iter > global_conf.decay_iter) then 
        global_conf.lrate = (global_conf.lrate * 0.6)
    end 
    if ppl_rec[iter].valid < ppl_last then 
        printf("%s PPL improves, saving net to file %s.%d...\n", global_conf.sche_log_pre, global_conf.param_fn, iter) 
        paramRepo:export(global_conf.param_fn .. '.' .. tostring(iter), nil) 
    else 
        printf("%s PPL did not improve, rejected, copying param file of last iter...\n", global_conf.sche_log_pre) 
        os.execute('cp ' .. global_conf.param_fn..'.'..tostring(iter - 1) .. ' ' .. global_conf.param_fn..'.'..tostring(iter))
    end 
    if ppl_last / ppl_rec[iter].valid < 1.0003 or lr_half == true then 
        lr_half = true
    end
    if ppl_rec[iter].valid < ppl_last then
        ppl_last = ppl_rec[iter].valid
    end
    printf("\n") 
    nerv.LMUtil.wait(2) 
end
printf("===VALIDATION PPL record===\n") 
for i, _ in pairs(ppl_rec) do 
    printf("<ITER%d LR%.5f train:%.3f valid:%.3f test:%.3f> \n", i, ppl_rec[i].lr, ppl_rec[i].train, ppl_rec[i].valid, ppl_rec[i].test) 
end 
printf("\n") 
printf("===FINAL TEST===\n") 
global_conf.sche_log_pre = "[SCHEDULER FINAL_TEST]:" 
tnn, paramRepo = load_net(global_conf, final_iter) 
LMTrainer.lm_process_file(global_conf, global_conf.test_fn, tnn, false) --false update!