nerv.include('reader.lua')
nerv.include('select_linear.lua')

gconf = {
    chunk_size = 15,
    dropout_rate = 0,
    lrate = 1.5,
    wcost = 1e-5,
    max_iter = 35,
    clip = 5,
    momentum = 0.9,
    batch_size = 200,
    test = true,
}

local hidden_size = 300
local vocab_size = 10000
local layer_num = 1
local dropout_rate = 0.5
local trainer = nerv.Trainer

function trainer:make_layer_repo(param_repo)
    local layers = {
        ['nerv.LSTMLayer'] = {},
        ['nerv.DropoutLayer'] = {},
        ['nerv.SelectLinearLayer'] = {
            ['select'] = {dim_in = {1}, dim_out = {hidden_size}, vocab = vocab_size, pr = param_repo},
        },
        ['nerv.AffineLayer'] = {
            output = {dim_in = {hidden_size}, dim_out = {vocab_size}, pr = param_repo},
        },
        ['nerv.SoftmaxCELayer'] = {
            softmax = {dim_in = {vocab_size, 1}, dim_out = {1}, compressed = true},
        },
    }
    for i = 1, layer_num do
        layers['nerv.LSTMLayer']['lstm' .. i] = {dim_in = {hidden_size}, dim_out = {hidden_size}, pr = param_repo}
        layers['nerv.DropoutLayer']['dropout' .. i] = {dim_in = {hidden_size}, dim_out = {hidden_size}}
    end
    return nerv.LayerRepo(layers, param_repo, gconf)
end

function trainer:get_network(layer_repo)
    local connections = {
        {'<input>[1]', 'select[1]', 0},
        {'select[1]', 'lstm1[1]', 0},
        {'dropout' .. layer_num .. '[1]', 'output[1]', 0},
        {'output[1]', 'softmax[1]', 0},
        {'<input>[2]', 'softmax[2]', 0},
        {'softmax[1]', '<output>[1]', 0},
    }
    for i = 1, layer_num do
        table.insert(connections, {'lstm' .. i .. '[1]', 'dropout' .. i .. '[1]', 0})
        if i < 1 then
            table.insert(connections, {'dropout' .. (i - 1) .. '[1]', 'lstm' .. i .. '[1]', 0})
        end
    end
    return nerv.GraphLayer('graph', gconf, {dim_in = {1, 1}, dim_out = {1}, layer_repo = layer_repo, connections = connections})
end

function trainer:get_input_order()
    return {'input', 'label'}
end

function trainer:get_readers(dataset)
    local data_path = 'nerv/nerv/examples/lmptb/PTBdata/'
    local vocab_file = data_path .. 'vocab'
    local train_file = data_path .. 'ptb.train.txt.adds'
    local cv_file = data_path .. 'ptb.valid.txt.adds'
    local test_file = data_path .. 'ptb.test.txt.adds'
    local reader
    if dataset == 'train' then  
        reader = nerv.Reader(vocab_file, train_file)
    elseif dataset == 'validate' then
        reader = nerv.Reader(vocab_file, cv_file)
    elseif dataset == 'test' then
        reader = nerv.Reader(vocab_file, test_file)
    else
        nerv.error('no such dataset')
    end
    return {{reader = reader, data = {input = 1, label = 1}}}
end

local total_err
local total_frame

function trainer:get_error()
    return math.pow(10, -total_err / total_frame)
end

function trainer:epoch_preprocess(dataset, do_train)
    if dataset == 'train' then
        gconf.dropout_rate = dropout_rate
        nerv.info('set dropout rate to %f', dropout_rate)
    end
    if dataset == 'validate' then
        gconf.dropout_rate = 0
        nerv.info('set dropout rate to 0')
    end
    if dataset == 'test' then
        gconf.dropout_rate = 0
        nerv.info('set dropout rate to 0')
    end
    total_err = 0
    total_frame = 0
end

function trainer:mini_batch_inprocess(cnt, info)
    for t = 1, gconf.chunk_size do
        local tmp = info.output[1][t]:new_to_host()
        for i = 1, gconf.batch_size do
            total_err = total_err + math.log10(math.exp(tmp[i - 1][0]))
        end
    end
    for i = 1, gconf.batch_size do
        total_frame = total_frame + info.seq_length[i]
    end
end