Merge branch 'liuq901-master' into seq-buffer

27 files changed, 557 insertions, 283 deletions

diff --git a/nerv/Makefile b/nerv/Makefile
index 68465a1..a5e4f66 100644
--- a/nerv/Makefile
+++ b/nerv/Makefile
@@ -44,7 +44,7 @@ LUA_LIBS := matrix/init.lua io/init.lua init.lua \
 			layer/elem_mul.lua layer/lstm.lua layer/lstm_gate.lua layer/dropout.lua layer/gru.lua \
 			layer/graph.lua layer/rnn.lua layer/duplicate.lua layer/identity.lua \
 			nn/init.lua nn/layer_repo.lua nn/param_repo.lua nn/network.lua \
-			io/sgd_buffer.lua io/seq_buffer.lua
+			io/frm_buffer.lua io/seq_buffer.lua
 
 INCLUDE := -I $(LUA_INCDIR) -DLUA_USE_APICHECK
 CUDA_INCLUDE := -I $(CUDA_BASE)/include/
diff --git a/nerv/examples/asr_trainer.lua b/nerv/examples/asr_trainer.lua
index 6bdf57c..645f1ef 100644
--- a/nerv/examples/asr_trainer.lua
+++ b/nerv/examples/asr_trainer.lua
@@ -22,9 +22,9 @@ local function build_trainer(ifname)
     local input_order = get_input_order()
 
     network = nerv.Network("nt", gconf, {network = network})
-    network:init(gconf.batch_size, 1)
+    network:init(gconf.batch_size, gconf.chunk_size)
     global_transf = nerv.Network("gt", gconf, {network = global_transf})
-    global_transf:init(gconf.batch_size, 1)
+    global_transf:init(gconf.batch_size, gconf.chunk_size)
 
     local iterative_trainer = function (prefix, scp_file, bp, rebind_param_repo)
         -- rebind the params if necessary
@@ -39,11 +39,17 @@ local function build_trainer(ifname)
         local buffer = make_buffer(make_readers(scp_file, layer_repo))
         -- initialize the network
         gconf.cnt = 0
-        err_input = {mat_type(gconf.batch_size, 1)}
-        err_input[1]:fill(1)
+        local err_input = {{}}
+        local output = {{}}
+        for i = 1, gconf.chunk_size do
+            local mini_batch = mat_type(gconf.batch_size, 1)
+            mini_batch:fill(1)
+            table.insert(err_input[1], mini_batch)
+            table.insert(output[1], mat_type(gconf.batch_size, 1))
+        end
         network:epoch_init()
         global_transf:epoch_init()
-        for data in buffer.get_data, buffer do
+        for d in buffer.get_data, buffer do
             -- prine stat periodically
             gconf.cnt = gconf.cnt + 1
             if gconf.cnt == 1000 then
@@ -54,35 +60,39 @@ local function build_trainer(ifname)
                 -- break
             end
             local input = {}
+            local err_output = {}
 --            if gconf.cnt == 1000 then break end
             for i, e in ipairs(input_order) do
                 local id = e.id
-                if data[id] == nil then
+                if d.data[id] == nil then
                     nerv.error("input data %s not found", id)
                 end
-                local transformed
+                local transformed = {}
+                local err_output_i = {}
                 if e.global_transf then
-                    transformed = nerv.speech_utils.global_transf(data[id],
-                                        global_transf,
-                                        gconf.frm_ext or 0, 0,
-                                        gconf)
+                    for _, mini_batch in ipairs(d.data[id]) do
+                        table.insert(transformed,
+                                        nerv.speech_utils.global_transf(mini_batch,
+                                            global_transf,
+                                            gconf.frm_ext or 0, 0,
+                                            gconf))
+                    end
                 else
-                    transformed = data[id]
+                    transformed = d.data[id]
+                end
+                for _, mini_batch in ipairs(transformed) do
+                    table.insert(err_output_i, mini_batch:create())
                 end
+                table.insert(err_output, err_output_i)
                 table.insert(input, transformed)
             end
-            local output = {mat_type(gconf.batch_size, 1)}
-            err_output = {}
-            for i = 1, #input do
-                table.insert(err_output, input[i]:create())
-            end
-            network:mini_batch_init({seq_length = table.vector(gconf.batch_size, 1),
-                                    new_seq = {},
+            network:mini_batch_init({seq_length = d.seq_length,
+                                    new_seq = d.new_seq,
                                     do_train = bp,
-                                    input = {input},
-                                    output = {output},
-                                    err_input = {err_input},
-                                    err_output = {err_output}})
+                                    input = input,
+                                    output = output,
+                                    err_input = err_input,
+                                    err_output = err_output})
             network:propagate()
             if bp then
                 network:back_propagate()
@@ -111,19 +121,21 @@ end
 
 local function check_and_add_defaults(spec, opts)
     local function get_opt_val(k)
-        return opts[string.gsub(k, '_', '-')].val
+        local k = string.gsub(k, '_', '-')
+        return opts[k].val, opts[k].specified
     end
     local opt_v = get_opt_val("resume_from")
     if opt_v then
+        nerv.info("resuming from previous training state")
         gconf = dofile(opt_v)
     else
         for k, v in pairs(spec) do
-            local opt_v = get_opt_val(k)
-            if opt_v ~= nil then
+            local opt_v, specified = get_opt_val(k)
+            if (not specified) and gconf[k] ~= nil then
+                nerv.info("using setting in network config file: %s = %s", k, gconf[k])
+            elseif opt_v ~= nil then
+                nerv.info("using setting in options: %s = %s", k, opt_v)
                 gconf[k] = opt_v
-            elseif gconf[k] ~= nil then
-            elseif v ~= nil then
-                gconf[k] = v
             end
         end
     end
@@ -168,6 +180,7 @@ end
 local trainer_defaults = {
     lrate = 0.8,
     batch_size = 256,
+    chunk_size = 1,
     buffer_size = 81920,
     wcost = 1e-6,
     momentum = 0.9,
diff --git a/nerv/examples/network_debug/config.lua b/nerv/examples/network_debug/config.lua
index e20d5a9..0429e9a 100644
--- a/nerv/examples/network_debug/config.lua
+++ b/nerv/examples/network_debug/config.lua
@@ -35,6 +35,10 @@ function get_layers(global_conf)
         ['nerv.SoftmaxCELayer'] = {
             softmax = {dim_in = {global_conf.vocab_size, global_conf.vocab_size}, dim_out = {1}, compressed = true},
         },
+        ['nerv.DuplicateLayer'] = {
+            dup1 = {dim_in = {1}, dim_out = {1}},
+            dup2 = {dim_in = {1}, dim_out = {1}},
+        },
     }
     for i = 1, global_conf.layer_num do
         layers['nerv.LSTMLayer']['lstm' .. i] = {dim_in = {global_conf.hidden_size}, dim_out = {global_conf.hidden_size}, pr = pr}
@@ -45,12 +49,14 @@ end
 
 function get_connections(global_conf)
     local connections = {
-        {'<input>[1]', 'select[1]', 0},
+        {'<input>[1]', 'dup1[1]', 0},
+        {'dup1[1]', 'select[1]', 0},
         {'select[1]', 'lstm1[1]', 0},
         {'dropout' .. global_conf.layer_num .. '[1]', 'output[1]', 0},
         {'output[1]', 'softmax[1]', 0},
         {'<input>[2]', 'softmax[2]', 0},
-        {'softmax[1]', '<output>[1]', 0},
+        {'softmax[1]', 'dup2[1]', 0},
+        {'dup2[1]', '<output>[1]', 0},
     }
     for i = 1, global_conf.layer_num do
         table.insert(connections, {'lstm' .. i .. '[1]', 'dropout' .. i .. '[1]', 0})
diff --git a/nerv/examples/network_debug/main.lua b/nerv/examples/network_debug/main.lua
index 790c404..bbcdb6c 100644
--- a/nerv/examples/network_debug/main.lua
+++ b/nerv/examples/network_debug/main.lua
@@ -6,35 +6,26 @@ nerv.include(arg[1])
 local global_conf = get_global_conf()
 local timer = global_conf.timer
 
-timer:tic('IO')
-
 local data_path = 'examples/lmptb/PTBdata/'
-local train_reader = nerv.Reader(data_path .. 'vocab', data_path .. 'ptb.train.txt.adds')
-local val_reader = nerv.Reader(data_path .. 'vocab', data_path .. 'ptb.valid.txt.adds')
-
-local train_data = train_reader:get_all_batch(global_conf)
-local val_data = val_reader:get_all_batch(global_conf)
 
 local layers = get_layers(global_conf)
 local connections = get_connections(global_conf)
 
-local NN = nerv.NN(global_conf, train_data, val_data, layers, connections)
-
-timer:toc('IO')
-timer:check('IO')
-io.flush()
+local NN = nerv.NN(global_conf, layers, connections)
 
 timer:tic('global')
 local best_cv = 1e10
 for i = 1, global_conf.max_iter do
     timer:tic('Epoch' .. i)
-    local train_ppl, val_ppl = NN:epoch()
+    local train_reader = nerv.Reader(data_path .. 'vocab', data_path .. 'ptb.train.txt.adds')
+    local val_reader = nerv.Reader(data_path .. 'vocab', data_path .. 'ptb.valid.txt.adds')
+    local train_ppl, val_ppl = NN:epoch(train_reader, val_reader)
+    nerv.printf('Epoch %d: %f %f %f\n', i, global_conf.lrate, train_ppl, val_ppl)
     if val_ppl < best_cv then
         best_cv = val_ppl
     else
         global_conf.lrate = global_conf.lrate / 2.0
     end
-    nerv.printf('Epoch %d: %f %f %f\n', i, global_conf.lrate, train_ppl, val_ppl)
     timer:toc('Epoch' .. i)
     timer:check('Epoch' .. i)
     io.flush()
@@ -43,3 +34,5 @@ timer:toc('global')
 timer:check('global')
 timer:check('network')
 timer:check('gc')
+timer:check('IO')
+global_conf.cumat_type.print_profile()
diff --git a/nerv/examples/network_debug/network.lua b/nerv/examples/network_debug/network.lua
index 5518e27..386c3b0 100644
--- a/nerv/examples/network_debug/network.lua
+++ b/nerv/examples/network_debug/network.lua
@@ -2,11 +2,17 @@ nerv.include('select_linear.lua')
 
 local nn = nerv.class('nerv.NN')
 
-function nn:__init(global_conf, train_data, val_data, layers, connections)
+function nn:__init(global_conf, layers, connections)
     self.gconf = global_conf
     self.network = self:get_network(layers, connections)
-    self.train_data = self:get_data(train_data)
-    self.val_data = self:get_data(val_data)
+
+    self.output = {}
+    self.err_output = {}
+    for i = 1, self.gconf.chunk_size do
+        self.output[i] = {self.gconf.cumat_type(self.gconf.batch_size, 1)}
+        self.err_output[i] = {self.gconf.cumat_type(self.gconf.batch_size, 1)}
+        self.err_output[i][2] = self.gconf.cumat_type(self.gconf.batch_size, 1)
+    end
 end
 
 function nn:get_network(layers, connections)
@@ -20,79 +26,67 @@ function nn:get_network(layers, connections)
     return network
 end
 
-function nn:get_data(data)
-    local err_output = {}
-    local softmax_output = {}
-    local output = {}
-    for i = 1, self.gconf.chunk_size do
-        err_output[i] = self.gconf.cumat_type(self.gconf.batch_size, 1)
-        softmax_output[i] = self.gconf.cumat_type(self.gconf.batch_size, self.gconf.vocab_size)
-        output[i] = self.gconf.cumat_type(self.gconf.batch_size, 1)
-    end
-    local ret = {}
-    for i = 1, #data do
-        ret[i] = {}
-        ret[i].input = {}
-        ret[i].output = {}
-        ret[i].err_input = {}
-        ret[i].err_output = {}
-        for t = 1, self.gconf.chunk_size do
-            ret[i].input[t] = {}
-            ret[i].output[t] = {}
-            ret[i].err_input[t] = {}
-            ret[i].err_output[t] = {}
-            ret[i].input[t][1] = data[i].input[t]
-            ret[i].input[t][2] = data[i].output[t]
-            ret[i].output[t][1] = output[t]
-            local err_input = self.gconf.mmat_type(self.gconf.batch_size, 1)
-            for j = 1, self.gconf.batch_size do
-                if t <= data[i].seq_len[j] then
-                    err_input[j - 1][0] = 1
-                else
-                    err_input[j - 1][0] = 0
+function nn:process(data, do_train, reader)
+    local timer = self.gconf.timer
+    local buffer = nerv.SeqBuffer(self.gconf, {
+        batch_size = self.gconf.batch_size, chunk_size = self.gconf.chunk_size,
+        readers = {reader},
+    })
+    local total_err = 0
+    local total_frame = 0
+    self.network:epoch_init()
+    while true do
+        timer:tic('IO')
+        data = buffer:get_data()
+        if data == nil then
+            break
+        end
+        local err_input = {}
+        if do_train then
+            for t = 1, self.gconf.chunk_size do
+                local tmp = self.gconf.mmat_type(self.gconf.batch_size, 1)
+                for i = 1, self.gconf.batch_size do
+                    if t <= data.seq_length[i] then
+                        tmp[i - 1][0] = 1
+                    else
+                        tmp[i - 1][0] = 0
+                    end
                 end
+                err_input[t] = {self.gconf.cumat_type.new_from_host(tmp)}
             end
-            ret[i].err_input[t][1] = self.gconf.cumat_type.new_from_host(err_input)
-            ret[i].err_output[t][1] = err_output[t]
-            ret[i].err_output[t][2] = softmax_output[t]
         end
-        ret[i].seq_length = data[i].seq_len
-        ret[i].new_seq = {}
-        for j = 1, self.gconf.batch_size do
-            if data[i].seq_start[j] then
-                table.insert(ret[i].new_seq, j)
-            end
+        local info = {input = {}, output = self.output, err_input = err_input, do_train = do_train,
+            err_output = self.err_output, seq_length = data.seq_length, new_seq = data.new_seq}
+        for t = 1, self.gconf.chunk_size do
+            info.input[t] = {data.data['input'][t]}
+            info.input[t][2] = data.data['label'][t]
         end
-    end
-    return ret
-end
+        timer:toc('IO')
 
-function nn:process(data, do_train)
-    local timer = self.gconf.timer
-    local total_err = 0
-    local total_frame = 0
-    self.network:epoch_init()
-    for id = 1, #data do
-        data[id].do_train = do_train
         timer:tic('network')
-        self.network:mini_batch_init(data[id])
+        self.network:mini_batch_init(info)
         self.network:propagate()
         timer:toc('network')
+
+        timer:tic('IO')
         for t = 1, self.gconf.chunk_size do
-            local tmp = data[id].output[t][1]:new_to_host()
+            local tmp = info.output[t][1]:new_to_host()
             for i = 1, self.gconf.batch_size do
-                if t <= data[id].seq_length[i] then
-                    total_err = total_err + math.log10(math.exp(tmp[i - 1][0]))
-                    total_frame = total_frame + 1
-                end
+                total_err = total_err + math.log10(math.exp(tmp[i - 1][0]))
             end
         end
+        for i = 1, self.gconf.batch_size do
+            total_frame = total_frame + info.seq_length[i]
+        end
+        timer:toc('IO')
+
+        timer:tic('network')
         if do_train then
-            timer:tic('network')
             self.network:back_propagate()
             self.network:update()
-            timer:toc('network')
         end
+        timer:toc('network')
+
         timer:tic('gc')
         collectgarbage('collect')
         timer:toc('gc')
@@ -100,11 +94,11 @@ function nn:process(data, do_train)
     return math.pow(10, - total_err / total_frame)
 end
 
-function nn:epoch()
-    local train_error = self:process(self.train_data, true)
+function nn:epoch(train_reader, val_reader)
+    local train_error = self:process(self.train_data, true, train_reader)
     local tmp = self.gconf.dropout_rate
     self.gconf.dropout_rate = 0
-    local val_error = self:process(self.val_data, false)
+    local val_error = self:process(self.val_data, false, val_reader)
     self.gconf.dropout_rate = tmp
     return train_error, val_error
 end
diff --git a/nerv/examples/network_debug/reader.lua b/nerv/examples/network_debug/reader.lua
index b10baaf..76a78cf 100644
--- a/nerv/examples/network_debug/reader.lua
+++ b/nerv/examples/network_debug/reader.lua
@@ -3,6 +3,7 @@ local Reader = nerv.class('nerv.Reader')
 function Reader:__init(vocab_file, input_file)
     self:get_vocab(vocab_file)
     self:get_seq(input_file)
+    self.offset = 1
 end
 
 function Reader:get_vocab(vocab_file)
@@ -32,6 +33,7 @@ function Reader:get_seq(input_file)
     local f = io.open(input_file, 'r')
     self.seq = {}
     while true do
+    -- for i = 1, 26 do
         local seq = f:read()
         if seq == nil then
             break
@@ -47,67 +49,19 @@ function Reader:get_seq(input_file)
     end
 end
 
-function Reader:get_in_out(id, pos)
-    return self.seq[id][pos], self.seq[id][pos + 1], pos + 1 == #self.seq[id]
-end
-
-function Reader:get_all_batch(global_conf)
-    local data = {}
-    local pos = {}
-    local offset = 1
-    for i = 1, global_conf.batch_size do
-        pos[i] = nil
+function Reader:get_data()
+    if self.offset > #self.seq then
+        return nil
     end
-    while true do
-    -- for i = 1, 26 do
-        local input = {}
-        local output = {}
-        for i = 1, global_conf.chunk_size do
-            input[i] = global_conf.mmat_type(global_conf.batch_size, 1)
-            input[i]:fill(global_conf.nn_act_default)
-            output[i] = global_conf.mmat_type(global_conf.batch_size, 1)
-            output[i]:fill(global_conf.nn_act_default)
-        end
-        local seq_start = {}
-        local seq_end = {}
-        local seq_len = {}
-        for i = 1, global_conf.batch_size do
-            seq_start[i] = false
-            seq_end[i] = false
-            seq_len[i] = 0
-        end
-        local has_new = false
-        for i = 1, global_conf.batch_size do
-            if pos[i] == nil then
-                if offset < #self.seq then
-                    seq_start[i] = true
-                    pos[i] = {offset, 1}
-                    offset = offset + 1
-                end
-            end
-            if pos[i] ~= nil then
-                has_new = true
-                for j = 1, global_conf.chunk_size do
-                    local final
-                    input[j][i-1][0], output[j][i-1][0], final = self:get_in_out(pos[i][1], pos[i][2])
-                    seq_len[i] = j
-                    if final then
-                        seq_end[i] = true
-                        pos[i] = nil
-                        break
-                    end
-                    pos[i][2] = pos[i][2] + 1
-                end
-            end
-        end
-        if not has_new then
-            break
-        end
-        for i = 1, global_conf.chunk_size do
-            input[i] = global_conf.cumat_type.new_from_host(input[i])
-            output[i] = global_conf.cumat_type.new_from_host(output[i])
-        end
-        table.insert(data, {input = input, output = output, seq_start = seq_start, seq_end = seq_end, seq_len = seq_len})
+    local tmp = self.seq[self.offset]
+    local res = {
+        input = nerv.MMatrixFloat(#tmp - 1, 1),
+        label = nerv.MMatrixFloat(#tmp - 1, 1),
+    }
+    for i = 1, #tmp - 1 do
+        res.input[i - 1][0] = tmp[i]
+        res.label[i - 1][0] = tmp[i + 1]
     end
-    return data
+    self.offset = self.offset + 1
+    return res
 end
diff --git a/nerv/examples/swb_baseline.lua b/nerv/examples/swb_baseline.lua
index 0ce8468..ece4d44 100644
--- a/nerv/examples/swb_baseline.lua
+++ b/nerv/examples/swb_baseline.lua
@@ -171,6 +171,7 @@ function make_buffer(readers)
         {
             buffer_size = gconf.buffer_size,
             batch_size = gconf.batch_size,
+            chunk_size = gconf.chunk_size,
             randomize = gconf.randomize,
             readers = readers,
             use_gpu = true
diff --git a/nerv/examples/swb_baseline2.lua b/nerv/examples/swb_baseline2.lua
index 6796f6f..38cfb9a 100644
--- a/nerv/examples/swb_baseline2.lua
+++ b/nerv/examples/swb_baseline2.lua
@@ -6,7 +6,8 @@ gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9, frm_ext = 5,
         cv_scp = "/speechlab/users/mfy43/swb50/train_cv.scp",
         htk_conf = "/speechlab/users/mfy43/swb50/plp_0_d_a.conf",
         initialized_param = {"/speechlab/users/mfy43/swb50/swb_init.nerv",
-                            "/speechlab/users/mfy43/swb50/swb_global_transf.nerv"}}
+                            "/speechlab/users/mfy43/swb50/swb_global_transf.nerv"},
+        chunk_size = 1}
 
 function make_layer_repo(param_repo)
     local layer_repo = nerv.LayerRepo(
@@ -145,7 +146,7 @@ end
 
 function make_readers(scp_file, layer_repo)
     return {
-                {reader = nerv.TNetReader(gconf,
+                {reader = nerv.HTKReader(gconf,
                     {
                         id = "main_scp",
                         scp_file = scp_file,
@@ -166,10 +167,11 @@ function make_readers(scp_file, layer_repo)
 end
 
 function make_buffer(readers)
-    return nerv.SGDBuffer(gconf,
+    return nerv.FrmBuffer(gconf,
         {
             buffer_size = gconf.buffer_size,
             batch_size = gconf.batch_size,
+            chunk_size = gconf.chunk_size,
             randomize = gconf.randomize,
             readers = readers,
             use_gpu = true
diff --git a/nerv/examples/timit_baseline2.lua b/nerv/examples/timit_baseline2.lua
index b1c1e66..658aa2e 100644
--- a/nerv/examples/timit_baseline2.lua
+++ b/nerv/examples/timit_baseline2.lua
@@ -8,7 +8,8 @@ gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9, frm_ext = 5,
                             "/speechlab/users/mfy43/timit/s5/exp/dnn4_nerv_dnn/nnet_output.nerv",
                             "/speechlab/users/mfy43/timit/s5/exp/dnn4_nerv_dnn/nnet_trans.nerv"},
          -- params in nnet_trans.nerv are included in the trained model
-        decode_param = {"/speechlab/users/mfy43/timit/s5/nerv_20160311205342/nnet_init_20160311211609_iter_13_lr0.013437_tr72.572_cv58.709.nerv"}}
+        decode_param = {"/speechlab/users/mfy43/timit/s5/nerv_20160311205342/nnet_init_20160311211609_iter_13_lr0.013437_tr72.572_cv58.709.nerv"},
+        chunk_size = 1}
 
 function make_layer_repo(param_repo)
     local layer_repo = nerv.LayerRepo(
@@ -176,10 +177,11 @@ function make_decode_readers(scp_file, layer_repo)
 end
 
 function make_buffer(readers)
-    return nerv.SGDBuffer(gconf,
+    return nerv.FrmBuffer(gconf,
         {
             buffer_size = gconf.buffer_size,
             batch_size = gconf.batch_size,
+            chunk_size = gconf.chunk_size,
             randomize = gconf.randomize,
             readers = readers,
             use_gpu = true
diff --git a/nerv/init.lua b/nerv/init.lua
index ff944b8..439a83e 100644
--- a/nerv/init.lua
+++ b/nerv/init.lua
@@ -109,7 +109,7 @@ function table.val_to_str(v)
                     (("number" == type(v) or
                     "string" == type(v) or
                     "boolean" == type(v)) and tostring(v)) or
-                    nil -- failed to serialize
+                    "" -- failed to serialize
     end
 end
 
@@ -226,7 +226,8 @@ function nerv.parse_args(argv, options, unordered)
         local opt_type = v[3]
         local opt_meta = {type = opt_type,
                             desc = v.desc or "",
-                            val = v.default}
+                            val = v.default,
+                            specified = false}
         if opt_short ~= nil then
             if type(opt_short) ~= "string" or #opt_short ~= 1 then err() end
             if opt_type ~= "boolean" then
@@ -246,6 +247,7 @@ function nerv.parse_args(argv, options, unordered)
                 for c in k:gmatch"." do
                     if sopts[c] then
                         sopts[c].val = true
+                        sopts[c].specified = true
                     else
                         nerv.error("invalid option -%s", c)
                     end
@@ -262,6 +264,7 @@ function nerv.parse_args(argv, options, unordered)
                                     k, opts[k].type)
                     else
                         opts[k].val = true
+                        opts[k].specified = true
                     end
                 else
                     local k, v = token:match(opt_with_val_exp)
@@ -269,6 +272,7 @@ function nerv.parse_args(argv, options, unordered)
                         if opts[k] == nil then
                             nerv.error("invalid option %s", token)
                         end
+                        opts[k].specified = true
                         if opts[k].type == "boolean" then
                             if v == "yes" then
                                 opts[k].val = true
diff --git a/nerv/io/sgd_buffer.lua b/nerv/io/frm_buffer.lua
index d78f6d1..9761f16 100644
--- a/nerv/io/sgd_buffer.lua
+++ b/nerv/io/frm_buffer.lua
@@ -1,6 +1,6 @@
-local SGDBuffer = nerv.class("nerv.SGDBuffer", "nerv.DataBuffer")
+local FrmBuffer = nerv.class("nerv.FrmBuffer", "nerv.DataBuffer")
 
-function SGDBuffer:__init(global_conf, buffer_conf)
+function FrmBuffer:__init(global_conf, buffer_conf)
     self.gconf = global_conf
     self.batch_size = buffer_conf.batch_size
     self.buffer_size = math.floor(buffer_conf.buffer_size /
@@ -57,7 +57,7 @@ function SGDBuffer:__init(global_conf, buffer_conf)
     end
 end
 
-function SGDBuffer:saturate()
+function FrmBuffer:saturate()
     local buffer_size = self.buffer_size
     self.head = 0
     self.tail = buffer_size
@@ -116,7 +116,7 @@ function SGDBuffer:saturate()
     return self.tail >= self.batch_size
 end
 
-function SGDBuffer:get_data()
+function FrmBuffer:get_data()
     local batch_size = self.batch_size
     if self.head >= self.tail then -- buffer is empty
         local t = os.clock()
@@ -132,7 +132,9 @@ function SGDBuffer:get_data()
         return nil -- the remaining data cannot build a batch
     end
     actual_batch_size = math.min(batch_size, self.tail - self.head)
-    local res = {}
+    local res = {seq_length = table.vector(gconf.batch_size, 1),
+                new_seq = {},
+                data = {}}
     for i, reader in ipairs(self.readers) do
         for id, buff in pairs(reader.buffs) do
             local batch = self.output_mat_type(actual_batch_size, buff.width)
@@ -141,7 +143,7 @@ function SGDBuffer:get_data()
             else
                 self.copy_from(batch, buff.data, self.head, self.head + actual_batch_size)
             end
-            res[id] = batch
+            res.data[id] = {batch}
         end
     end
     self.head = self.head + actual_batch_size
diff --git a/nerv/io/init.lua b/nerv/io/init.lua
index c36d850..d3ba27c 100644
--- a/nerv/io/init.lua
+++ b/nerv/io/init.lua
@@ -56,5 +56,5 @@ function DataBuffer:get_data()
     nerv.error_method_not_implemented()
 end
 
-nerv.include('sgd_buffer.lua')
+nerv.include('frm_buffer.lua')
 nerv.include('seq_buffer.lua')
diff --git a/nerv/io/seq_buffer.lua b/nerv/io/seq_buffer.lua
index e69de29..029e7b8 100644
--- a/nerv/io/seq_buffer.lua
+++ b/nerv/io/seq_buffer.lua
@@ -0,0 +1,108 @@
+local SeqBuffer = nerv.class('nerv.SeqBuffer', 'nerv.DataBuffer')
+
+function SeqBuffer:__init(global_conf, buffer_conf)
+    self.gconf = global_conf
+
+    self.batch_size = buffer_conf.batch_size
+    self.chunk_size = buffer_conf.chunk_size
+    self.readers = {}
+    for _, v in ipairs(buffer_conf.readers) do
+        table.insert(self.readers, v.reader)
+    end
+    self.nn_act_default = buffer_conf.nn_act_default
+    if self.nn_act_default == nil then
+        self.nn_act_default = 0
+    end
+
+    self.mat_type = self.gconf.mmat_type
+    self.queue = {}
+    self.head = 1
+    self.tail = 0
+end
+
+function SeqBuffer:new_mini_batch()
+    local res = {}
+    res.data = {}
+    res.new_seq = {}
+    res.seq_length = {}
+    for i = 1, self.batch_size do
+        res.seq_length[i] = 0
+    end
+    return res
+end
+
+function SeqBuffer:saturate(batch)
+    if self.queue[self.head] ~= nil and self.queue[self.head].seq_length[batch] ~= 0 then
+