Merge branch 'liuq901-master'

18 files changed, 869 insertions, 1145 deletions

diff --git a/nerv/Makefile b/nerv/Makefile
index e8bcad2..7921bd9 100644
--- a/nerv/Makefile
+++ b/nerv/Makefile
@@ -42,9 +42,9 @@ LUA_LIBS := matrix/init.lua io/init.lua init.lua \
 			layer/init.lua layer/affine.lua layer/sigmoid.lua layer/tanh.lua layer/softmax_ce.lua layer/softmax.lua \
 			layer/window.lua layer/bias.lua layer/combiner.lua layer/mse.lua \
 			layer/elem_mul.lua layer/lstm.lua layer/lstm_gate.lua layer/dropout.lua layer/gru.lua \
-			nn/init.lua nn/layer_repo.lua nn/param_repo.lua nn/layer_dag.lua \
-			io/sgd_buffer.lua \
-			tnn/init.lua tnn/sutil.lua tnn/tnn.lua
+			layer/graph.lua layer/rnn.lua layer/duplicate.lua layer/identity.lua \
+			nn/init.lua nn/layer_repo.lua nn/param_repo.lua nn/network.lua \
+			io/sgd_buffer.lua
 
 INCLUDE := -I $(LUA_INCDIR) -DLUA_USE_APICHECK
 CUDA_INCLUDE := -I $(CUDA_BASE)/include/
diff --git a/nerv/examples/asr_trainer.lua b/nerv/examples/asr_trainer.lua
index 5bf28bd..6bdf57c 100644
--- a/nerv/examples/asr_trainer.lua
+++ b/nerv/examples/asr_trainer.lua
@@ -20,6 +20,12 @@ local function build_trainer(ifname)
     local network = get_network(layer_repo)
     local global_transf = get_global_transf(layer_repo)
     local input_order = get_input_order()
+
+    network = nerv.Network("nt", gconf, {network = network})
+    network:init(gconf.batch_size, 1)
+    global_transf = nerv.Network("gt", gconf, {network = global_transf})
+    global_transf:init(gconf.batch_size, 1)
+
     local iterative_trainer = function (prefix, scp_file, bp, rebind_param_repo)
         -- rebind the params if necessary
         if rebind_param_repo then
@@ -32,10 +38,11 @@ local function build_trainer(ifname)
         -- build buffer
         local buffer = make_buffer(make_readers(scp_file, layer_repo))
         -- initialize the network
-        network:init(gconf.batch_size)
         gconf.cnt = 0
         err_input = {mat_type(gconf.batch_size, 1)}
         err_input[1]:fill(1)
+        network:epoch_init()
+        global_transf:epoch_init()
         for data in buffer.get_data, buffer do
             -- prine stat periodically
             gconf.cnt = gconf.cnt + 1
@@ -69,10 +76,17 @@ local function build_trainer(ifname)
             for i = 1, #input do
                 table.insert(err_output, input[i]:create())
             end
-            network:propagate(input, output)
+            network:mini_batch_init({seq_length = table.vector(gconf.batch_size, 1),
+                                    new_seq = {},
+                                    do_train = bp,
+                                    input = {input},
+                                    output = {output},
+                                    err_input = {err_input},
+                                    err_output = {err_output}})
+            network:propagate()
             if bp then
-                network:back_propagate(err_input, err_output, input, output)
-                network:update(err_input, input, output)
+                network:back_propagate()
+                network:update()
             end
             -- collect garbage in-time to save GPU memory
             collectgarbage("collect")
diff --git a/nerv/examples/timit_baseline2.lua b/nerv/examples/timit_baseline2.lua
index 2d144b5..d783c3d 100644
--- a/nerv/examples/timit_baseline2.lua
+++ b/nerv/examples/timit_baseline2.lua
@@ -61,35 +61,35 @@ function make_layer_repo(param_repo)
 
     layer_repo:add_layers(
     {
-        ["nerv.DAGLayer"] =
+        ["nerv.GraphLayer"] =
         {
             global_transf = {
                 dim_in = {440}, dim_out = {440},
-                sub_layers = layer_repo,
+                layer_repo = layer_repo,
                 connections = {
-                    ["<input>[1]"] = "blayer1[1]",
-                    ["blayer1[1]"] = "wlayer1[1]",
-                    ["wlayer1[1]"] = "<output>[1]"
+                    {"<input>[1]", "blayer1[1]", 0},
+                    {"blayer1[1]", "wlayer1[1]", 0},
+                    {"wlayer1[1]", "<output>[1]", 0}
                 }
             },
             main = {
                 dim_in = {440}, dim_out = {1959},
-                sub_layers = layer_repo,
+                layer_repo = layer_repo,
                 connections = {
-                    ["<input>[1]"] = "affine0[1]",
-                    ["affine0[1]"] = "sigmoid0[1]",
-                    ["sigmoid0[1]"] = "affine1[1]",
-                    ["affine1[1]"] = "sigmoid1[1]",
-                    ["sigmoid1[1]"] = "affine2[1]",
-                    ["affine2[1]"] = "sigmoid2[1]",
-                    ["sigmoid2[1]"] = "affine3[1]",
-                    ["affine3[1]"] = "sigmoid3[1]",
-                    ["sigmoid3[1]"] = "affine4[1]",
-                    ["affine4[1]"] = "sigmoid4[1]",
-                    ["sigmoid4[1]"] = "affine5[1]",
-                    ["affine5[1]"] = "sigmoid5[1]",
-                    ["sigmoid5[1]"] = "affine6[1]",
-                    ["affine6[1]"] = "<output>[1]"
+                    {"<input>[1]", "affine0[1]", 0},
+                    {"affine0[1]", "sigmoid0[1]", 0},
+                    {"sigmoid0[1]", "affine1[1]", 0},
+                    {"affine1[1]", "sigmoid1[1]", 0},
+                    {"sigmoid1[1]", "affine2[1]", 0},
+                    {"affine2[1]", "sigmoid2[1]", 0},
+                    {"sigmoid2[1]", "affine3[1]", 0},
+                    {"affine3[1]", "sigmoid3[1]", 0},
+                    {"sigmoid3[1]", "affine4[1]", 0},
+                    {"affine4[1]", "sigmoid4[1]", 0},
+                    {"sigmoid4[1]", "affine5[1]", 0},
+                    {"affine5[1]", "sigmoid5[1]", 0},
+                    {"sigmoid5[1]", "affine6[1]", 0},
+                    {"affine6[1]", "<output>[1]", 0}
                 }
             }
         }
@@ -97,25 +97,25 @@ function make_layer_repo(param_repo)
 
     layer_repo:add_layers(
     {
-        ["nerv.DAGLayer"] =
+        ["nerv.GraphLayer"] =
         {
             ce_output = {
                 dim_in = {440, 1}, dim_out = {1},
-                sub_layers = layer_repo,
+                layer_repo = layer_repo,
                 connections = {
-                    ["<input>[1]"] = "main[1]",
-                    ["main[1]"] = "ce_crit[1]",
-                    ["<input>[2]"] = "ce_crit[2]",
-                    ["ce_crit[1]"] = "<output>[1]"
+                    {"<input>[1]", "main[1]", 0},
+                    {"main[1]", "ce_crit[1]", 0},
+                    {"<input>[2]", "ce_crit[2]", 0},
+                    {"ce_crit[1]", "<output>[1]", 0}
                 }
             },
             softmax_output = {
                 dim_in = {440}, dim_out = {1959},
-                sub_layers = layer_repo,
+                layer_repo = layer_repo,
                 connections = {
-                    ["<input>[1]"] = "main[1]",
-                    ["main[1]"] = "softmax[1]",
-                    ["softmax[1]"] = "<output>[1]"
+                    {"<input>[1]",  "main[1]", 0},
+                    {"main[1]", "softmax[1]", 0},
+                    {"softmax[1]", "<output>[1]", 0}
                 }
             }
         }
diff --git a/nerv/init.lua b/nerv/init.lua
index da7df29..ff944b8 100644
--- a/nerv/init.lua
+++ b/nerv/init.lua
@@ -347,10 +347,18 @@ function table.extend(tbl1, tbl2)
     end
 end
 
+function table.vector(len, fill)
+    local v = {}
+    fill = fill or 0
+    for i = 1, len do
+        table.insert(v, fill)
+    end
+    return v
+end
+
 -- the following lines trigger the initialization of basic modules
 
 nerv.include('matrix/init.lua')
 nerv.include('io/init.lua')
 nerv.include('layer/init.lua')
 nerv.include('nn/init.lua')
-nerv.include('tnn/init.lua')
diff --git a/nerv/layer/duplicate.lua b/nerv/layer/duplicate.lua
new file mode 100644
index 0000000..137472b
--- /dev/null
+++ b/nerv/layer/duplicate.lua
@@ -0,0 +1,41 @@
+local DuplicateLayer = nerv.class('nerv.DuplicateLayer', 'nerv.Layer')
+
+function DuplicateLayer:__init(id, global_conf, layer_conf)
+    nerv.Layer.__init(self, id, global_conf, layer_conf)
+    self:check_dim_len(1, -1)
+    if #self.dim_out < 1 then
+        nerv.error('no output specified')
+    end
+    for i = 1, #self.dim_out do
+        if self.dim_out[i] ~= self.dim_in[1] then
+            nerv.error('mismatching dimensions of outputs')
+        end
+    end
+end
+
+function DuplicateLayer:init()
+end
+
+function DuplicateLayer:batch_resize()
+end
+
+function DuplicateLayer:propagate(input, output)
+    for i = 1, #self.dim_out do
+        output[i]:copy_from(input[1])
+        -- FIXME: use reference copy to speed up
+    end
+end
+
+function DuplicateLayer:back_propagate(bp_err, next_bp_err)
+    next_bp_err[1]:copy_from(bp_err[1])
+    for i = 2, #self.dim_out do
+        next_bp_err[1]:add(next_bp_err[1], bp_err[i], 1.0, 1.0)
+    end
+end
+
+function DuplicateLayer:update()
+end
+
+function DuplicateLayer:get_params()
+    return nerv.ParamRepo({}, self.loc_type)
+end
diff --git a/nerv/layer/graph.lua b/nerv/layer/graph.lua
new file mode 100644
index 0000000..5f42fca
--- /dev/null
+++ b/nerv/layer/graph.lua
@@ -0,0 +1,156 @@
+local GraphLayer = nerv.class('nerv.GraphLayer', 'nerv.Layer')
+
+function GraphLayer:__init(id, global_conf, layer_conf)
+    nerv.Layer.__init(self, id, global_conf, layer_conf)
+    self:graph_init(layer_conf.layer_repo, layer_conf.connections)
+end
+
+local function parse_id(str)
+    local id, port, _
+    _, _, id, port = string.find(str, "([a-zA-Z0-9_.]+)%[([0-9]+)%]")
+    if id == nil or port == nil then
+        _, _, id, port = string.find(str, "(.+)%[([0-9]+)%]")
+        if not (id == "<input>" or id == "<output>") then
+            nerv.error("wrong format of connection id")
+        end
+    end
+    port = tonumber(port)
+    return id, port
+end
+
+function GraphLayer:add_prefix(layers, connections)
+    local function ap(name)
+        return self.id .. '.' .. name
+    end
+
+    for layer_type, sublayers in pairs(layers) do
+        local tmp = {}
+        for name, layer_config in pairs(sublayers) do
+            tmp[ap(name)] = layer_config
+        end
+        layers[layer_type] = tmp
+    end
+
+    for i = 1, #connections do
+        local from, to = connections[i][1], connections[i][2]
+        if parse_id(from) ~= '<input>' then
+            connections[i][1] = ap(from)
+        end
+        if parse_id(to) ~= '<output>' then
+            connections[i][2] = ap(to)
+        end
+    end
+end
+
+function GraphLayer:discover(id, layer_repo)
+    if id == '<output>' then
+        id = '<input>'
+    end
+    local layers = self.layers
+    local ref = layers[id]
+    if ref == nil then
+        local layer = layer_repo:get_layer(id)
+        local dim_in, dim_out = layer:get_dim()
+        self.layer_num = self.layer_num + 1
+        ref = {
+            layer = layer,
+            inputs = {},
+            outputs = {},
+            dim_in = dim_in,
+            dim_out = dim_out,
+            id = self.layer_num,
+        }
+        layers[id] = ref
+    end
+    return ref
+end
+
+function GraphLayer:graph_init(layer_repo, connections)
+    local layers = {}
+    layers['<input>'] = {
+        inputs = {},
+        outputs = {},
+        dim_in = self.dim_out,
+        dim_out = self.dim_in,
+        id = 0,
+    }
+    self.layers = layers
+    self.layer_num = 0
+    self.connections = {}
+
+    -- check data dimension between connected ports
+    for _, edge in pairs(connections) do
+        local from, to, time = edge[1], edge[2], edge[3]
+        local id_from, port_from = parse_id(from)
+        local id_to, port_to = parse_id(to)
+        local ref_from = self:discover(id_from, layer_repo)
+        local ref_to = self:discover(id_to, layer_repo)
+        if ref_from.outputs[port_from] ~= nil then
+            nerv.error('%s has already been attached', from)
+        end
+        if ref_to.inputs[port_to] ~= nil then
+            nerv.error('%s has already been attached', to)
+        end
+        if ref_from.dim_out[port_from] ~= ref_to.dim_in[port_to] then
+            nerv.error('mismatching data dimension between %s and %s', from, to)
+        end
+        if ref_from.id == 0 and ref_to.id == 0 then
+            nerv.error('short-circuit connection between <input> and <output>')
+        end
+        ref_from.outputs[port_from] = true
+        ref_to.inputs[port_to] = true
+        table.insert(self.connections, {ref_from.id, port_from, ref_to.id, port_to, time})
+    end
+
+    -- check dangling ports
+    for id, ref in pairs(layers) do
+        if id ~= '<input>' then
+            for i = 1, #ref.dim_in do
+                if ref.inputs[i] == nil then
+                    nerv.error('dangling input port %d of layer %s', i, id)
+                end
+            end
+            for i = 1, #ref.dim_out do
+                if ref.outputs[i] == nil then
+                   nerv.error('dangling output port %d os layer %s', i, id)
+                end
+            end
+        end
+    end
+    for i = 1, #self.dim_in do
+        if layers['<input>'].outputs[i] == nil then
+            nerv.error('dangling port %d of layer <input>', i)
+        end
+    end
+    for i = 1, #self.dim_out do
+        if layers['<input>'].inputs[i] == nil then
+            nerv.error('dangling port %d of layer <output>', i)
+        end
+    end
+end
+
+function GraphLayer:set_attr(name, value)
+    self[name] = value
+    for id, ref in pairs(self.layers) do
+        if id ~= '<input>' then
+            ref.layer:set_attr(name, value)
+        end
+    end
+end
+
+function GraphLayer:get_sublayer(id)
+    if self.layers[id] == nil or id == '<input>' then
+        nerv.error('layer with id %s not found', id)
+    end
+    return self.layers[id].layer
+end
+
+function GraphLayer:get_params()
+    local param_repos = {}
+    for id, ref in pairs(self.layers) do
+        if id ~= '<input>' then
+            table.insert(param_repos, ref.layer:get_params())
+        end
+    end
+    return nerv.ParamRepo.merge(param_repos, self.loc_type)
+end
diff --git a/nerv/layer/gru.lua b/nerv/layer/gru.lua
index a590a67..71718d7 100644
--- a/nerv/layer/gru.lua
+++ b/nerv/layer/gru.lua
@@ -13,7 +13,7 @@ function GRULayer:__init(id, global_conf, layer_conf)
     -- prepare a DAGLayer to hold the lstm structure
     local pr = layer_conf.pr
     if pr == nil then
-        pr = nerv.ParamRepo(nil, self.loc_type)
+        pr = nerv.ParamRepo({}, self.loc_type)
     end
     
     local function ap(str)
@@ -102,7 +102,7 @@ end
 function GRULayer:bind_params()
     local pr = layer_conf.pr
     if pr == nil then
-        pr = nerv.ParamRepo(nil, self.loc_type)
+        pr = nerv.ParamRepo({}, self.loc_type)
     end
     self.lrepo:rebind(pr)
 end
diff --git a/nerv/layer/identity.lua b/nerv/layer/identity.lua
new file mode 100644
index 0000000..d56337d
--- /dev/null
+++ b/nerv/layer/identity.lua
@@ -0,0 +1,30 @@
+local IdentityLayer = nerv.class('nerv.IdentityLayer', 'nerv.Layer')
+
+function IdentityLayer:__init(id, global_conf, layer_conf)
+    nerv.Layer.__init(self, id, global_conf, layer_conf)
+    self:check_dim_len(1, 1)
+    if self.dim_in[1] ~= self.dim_out[1] then
+        nerv.error('mismatching dimensions of input and output')
+    end
+end
+
+function IdentityLayer:init()
+end
+
+function IdentityLayer:batch_resize()
+end
+
+function IdentityLayer:propagate(input, output)
+    output[1]:copy_from(input[1])
+end
+
+function IdentityLayer:back_propagate(bp_err, next_bp_err)
+    next_bp_err[1]:copy_from(bp_err[1])
+end
+
+function IdentityLayer:update()
+end
+
+function IdentityLayer:get_params()
+    return nerv.ParamRepo({}, self.loc_type)
+end
diff --git a/nerv/layer/init.lua b/nerv/layer/init.lua
index 146ad8c..475ef62 100644
--- a/nerv/layer/init.lua
+++ b/nerv/layer/init.lua
@@ -85,6 +85,14 @@ function Layer:get_dim()
     return self.dim_in, self.dim_out
 end
 
+function Layer:set_attr(name, value)
+    self[name] = value
+end
+
+function Layer:get_sublayer(id)
+    nerv.error('primitive layer does not have sublayers')
+end
+
 function Layer:find_param(plist, lconf, gconf, p_type, p_dim)
     if type(plist) == "string" then
         plist = {plist}
@@ -119,6 +127,7 @@ function Layer:find_param(plist, lconf, gconf, p_type, p_dim)
     return p
 end
 
+nerv.include('graph.lua')
 nerv.include('affine.lua')
 nerv.include('sigmoid.lua')
 nerv.include('tanh.lua')
@@ -133,6 +142,9 @@ nerv.include('lstm.lua')
 nerv.include('lstm_gate.lua')
 nerv.include('dropout.lua')
 nerv.include('gru.lua')
+nerv.include('rnn.lua')
+nerv.include('duplicate.lua')
+nerv.include('identity.lua')
 
 -- The following lines are for backward compatibility, and will be removed in
 -- the future. The use of these names are deprecated.
diff --git a/nerv/layer/lstm.lua b/nerv/layer/lstm.lua
index d4c9212..641d5dc 100644
--- a/nerv/layer/lstm.lua
+++ b/nerv/layer/lstm.lua
@@ -8,7 +8,7 @@ function LSTMLayer:__init(id, global_conf, layer_conf)
     -- prepare a DAGLayer to hold the lstm structure
     local pr = layer_conf.pr
     if pr == nil then
-        pr = nerv.ParamRepo(nil, self.loc_type)
+        pr = nerv.ParamRepo({}, self.loc_type)
     end
     
     local function ap(str)
@@ -18,47 +18,47 @@ function LSTMLayer:__init(id, global_conf, layer_conf)
     local dout1, dout2, dout3 = self.dim_out[1], self.dim_out[2], self.dim_out[3]
     local layers = {
         ["nerv.CombinerLayer"] = {
-            [ap("inputXDup")] = {{}, {dim_in = {din1},
+            [ap("inputXDup")] = {dim_in = {din1},
                                       dim_out = {din1, din1, din1, din1},
-                                      lambda = {1}}},
+                                      lambda = {1}},
 
-            [ap("inputHDup")] = {{}, {dim_in = {din2},
+            [ap("inputHDup")] = {dim_in = {din2},
                                       dim_out = {din2, din2, din2, din2},
-                                      lambda = {1}}},
+                                      lambda = {1}},
 
-            [ap("inputCDup")] = {{}, {dim_in = {din3},
+            [ap("inputCDup")] = {dim_in = {din3},
                                       dim_out = {din3, din3, din3},
-                                      lambda = {1}}},
+                                      lambda = {1}},
 
-            [ap("mainCDup")] = {{}, {dim_in = {din3, din3},
+            [ap("mainCDup")] = {dim_in = {din3, din3},
                                      dim_out = {din3, din3, din3},
-                                     lambda = {1, 1}}},
+                                     lambda = {1, 1}},
         },
         ["nerv.AffineLayer"] = {
-            [ap("mainAffineL")] = {{}, {dim_in = {din1, din2},
+            [ap("mainAffineL")] = {dim_in = {din1, din2},
                                         dim_out = {dout1},
-                                        pr = pr}},
+                                        pr = pr},
         },
         ["nerv.TanhLayer"] = {
-            [ap("mainTanhL")] = {{}, {dim_in = {dout1}, dim_out = {dout1}}},
-            [ap("outputTanhL")] = {{}, {dim_in = {dout1}, dim_out = {dout1}}},
+            [ap("mainTanhL")] = {dim_in = {dout1}, dim_out = {dout1}},
+            [ap("outputTanhL")] = {dim_in = {dout1}, dim_out = {dout1}},
         },
         ["nerv.LSTMGateLayer"] = {
-            [ap("forgetGateL")] = {{}, {dim_in = {din1, din2, din3},
-                                        dim_out = {din3}, pr = pr}},
-            [ap("inputGateL")] = {{}, {dim_in = {din1, din2, din3},
-                                        dim_out = {din3}, pr = pr}},
-            [ap("outputGateL")] = {{}, {dim_in = {din1, din2, din3},
-                                        dim_out = {din3}, pr = pr}},
+            [ap("forgetGateL")] = {dim_in = {din1, din2, din3},
+                                        dim_out = {din3}, pr = pr},
+            [ap("inputGateL")] = {dim_in = {din1, din2, din3},
+                                        dim_out = {din3}, pr = pr},
+            [ap("outputGateL")] = {dim_in = {din1, din2, din3},
+                                        dim_out = {din3}, pr = pr},
 
         },
         ["nerv.ElemMulLayer"] = {
-            [ap("inputGMulL")] = {{}, {dim_in = {din3, din3},
-                                        dim_out = {din3}}},
-            [ap("forgetGMulL")] = {{}, {dim_in = {din3, din3},
-                                        dim_out = {din3}}},
-            [ap("outputGMulL")] = {{}, {dim_in = {din3, din3},
-                                        dim_out = {din3}}},
+            [ap("inputGMulL")] = {dim_in = {din3, din3},
+                                        dim_out = {din3}},
+            [ap("forgetGMulL")] = {dim_in = {din3, din3},
+                                        dim_out = {din3}},
+            [ap("outputGMulL")] = {dim_in = {din3, din3},
+                                        dim_out = {din3}},
         },
     }
     
@@ -114,7 +114,7 @@ end
 function LSTMLayer:bind_params()
     local pr = layer_conf.pr
     if pr == nil then
-        pr = nerv.ParamRepo(nil, self.loc_type)
+        pr = nerv.ParamRepo({}, self.loc_type)
     end
     self.lrepo:rebind(pr)
 end
diff --git a/nerv/layer/rnn.lua b/nerv/layer/rnn.lua
new file mode 100644
index 0000000..e59cf5b
--- /dev/null
+++ b/nerv/layer/rnn.lua
@@ -0,0 +1,38 @@
+local RNNLayer = nerv.class('nerv.RNNLayer', 'nerv.GraphLayer')
+
+function RNNLayer:__init(id, global_conf, layer_conf)
+    nerv.Layer.__init(self, id, global_conf, layer_conf)
+    self:check_dim_len(1, 1)
+
+    local din = layer_conf.dim_in[1]
+    local dout = layer_conf.dim_out[1]
+
+    local pr = layer_conf.pr
+    if pr == nil then
+        pr = nerv.ParamRepo({}, self.loc_type)
+    end
+
+    local layers = {
+        ['nerv.AffineLayer'] = {
+            main = {dim_in = {din, dout}, dim_out = {dout}, pr = pr},
+        },
+        ['nerv.SigmoidLayer'] = {
+            sigmoid = {dim_in = {dout}, dim_out = {dout}},
+        },
+        ['nerv.DuplicateLayer'] = {
+            dup = {dim_in = {dout}, dim_out = {dout, dout}},
+        }
+    }
+
+    local connections = {
+        {'<input>[1]', 'main[1]', 0},
+        {'main[1]', 'sigmoid[1]', 0},
+        {'sigmoid[1]', 'dup[1]', 0},
+        {'dup[1]', 'main[2]', 1},
+        {'dup[2]', '<output>[1]', 0},
+    }
+
+    self:add_prefix(layers, connections)
+    local layer_repo = nerv.LayerRepo(layers, pr, global_conf)
+    self:graph_init(layer_repo, connections)
+end
diff --git a/nerv/layer/sigmoid.lua b/nerv/layer/sigmoid.lua
index a9f9749..5974ffc 100644
--- a/nerv/layer/sigmoid.lua
+++ b/nerv/layer/sigmoid.lua
@@ -3,6 +3,9 @@ local SigmoidLayer = nerv.class("nerv.SigmoidLayer", "nerv.Layer")
 function SigmoidLayer:__init(id, global_conf, layer_conf)
     nerv.Layer.__init(self, id, global_conf, layer_conf)
     self:check_dim_len(1, 1)
+    if self.dim_in[1] ~= self.dim_out[1] then
+        nerv.error("mismatching dimensions of input and output")
+    end
 end
 
 function SigmoidLayer:bind_params()
@@ -10,9 +13,6 @@ function SigmoidLayer:bind_params()
 end
 
 function SigmoidLayer:init()
-    if self.dim_in[1] ~= self.dim_out[1] then
-        nerv.error("mismatching dimensions of input and output")
-    end
 end
 
 function SigmoidLayer:batch_resize(batch_size)
diff --git a/nerv/nn/init.lua b/nerv/nn/init.lua
index cbaf52b..1037d05 100644
--- a/nerv/nn/init.lua
+++ b/nerv/nn/init.lua
@@ -1,3 +1,3 @@
 nerv.include('layer_repo.lua')
 nerv.include('param_repo.lua')
-nerv.include('layer_dag.lua')
+nerv.include('network.lua')
diff --git a/nerv/nn/layer_dag.lua b/nerv/nn/layer_dag.lua
deleted file mode 100644
index f999752..0000000
--- a/nerv/nn/layer_dag.lua
+++ /dev/null
@@ -1,352 +0,0 @@
-local DAGLayer = nerv.class("nerv.DAGLayer", "nerv.Layer")
-
-local function parse_id(str)
-    local id, port, _
-    _, _, id, port = string.find(str, "([a-zA-Z0-9_.]+)%[([0-9]+)%]")
-    if id == nil or port == nil then
-        _, _, id, port = string.find(str, "(.+)%[([0-9]+)%]")
-        if not (id == "<input>" or id == "<output>") then
-            nerv.error("wrong format of connection id")
-        end
-    end
-    port = tonumber(port)
-    return id, port
-end
-
-local function discover(id, layers, layer_repo)
-    local ref = layers[id]
-    if id == "<input>" or id == "<output>" then
-        return nil
-    end
-    if ref == nil then
-        local layer = layer_repo:get_layer(id)
-        local dim_in, dim_out = layer:get_dim()
-        ref = {
-            layer = layer,
-            inputs = {},
-            outputs = {},
-            err_inputs = {},
-            err_outputs = {},
-            next_layers = {},
-            input_len = #dim_in,
-            output_len = #dim_out,
-            in_deg = 0,
-            visited = false
-        }
-        layers[id] = ref
-    end
-    return ref
-end
-
-local function touch_list_by_idx(list, idx)
-    if list[idx] == nil then
-        list[idx] = {}
-    end
-end
-
-function DAGLayer:__init(id, global_conf, layer_conf)
-    local layers = {}
-    local inputs = {}
-    local outputs = {}
-    local dim_in = layer_conf.dim_in
-    local dim_out = layer_conf.dim_out
-    local parsed_conn = {}
-    for from, to in pairs(layer_conf.connections) do
-        local id_from, port_from = parse_id(from)
-        local id_to, port_to = parse_id(to)
-        local ref_from = discover(id_from, layers, layer_conf.sub_layers)
-        local ref_to = discover(id_to, layers, layer_conf.sub_layers)
-        local input_dim, output_dim, _
-        if ref_from then
-            touch_list_by_idx(ref_from.outputs, 1)
-            if ref_from.outputs[1][port_from] ~= nil then
-                nerv.error("%s has already been attached", from)
-            end
-        end
-        if ref_to then
-            touch_list_by_idx(ref_to.inputs, 1)
-            if ref_to.inputs[1][port_to] ~= nil then
-                nerv.error("%s has already been attached", to)
-            end
-        end
-        if id_from == "<input>" then
-            input_dim, _ = ref_to.layer:get_dim()
-            if dim_in[port_from] ~= input_dim[port_to] then
-                nerv.error("mismatching data dimension between %s and %s", from, to)
-            end
-            inputs[port_from] = {ref_to, port_to}
-            ref_to.inputs[1][port_to] = inputs -- just a place holder
-        elseif id_to == "<output>" then
-            _, output_dim = ref_from.layer:get_dim()
-            if output_dim[port_from] ~= dim_out[port_to] then
-                nerv.error("mismatching data dimension between %s and %s", from, to)
-            end
-            outputs[port_to] = {ref_from, port_from}
-            ref_from.outputs[1][port_from] = outputs -- just a place holder
-        else
-            _, output_dim = ref_from.layer:get_dim()
-            input_dim, _ = ref_to.layer:get_dim()
-            if output_dim[port_from] ~= input_dim[port_to] then
-                nerv.error("mismatching data dimension between %s and %s", from, to)
-            end
-
-            table.insert(parsed_conn,
-            {{ref_from, port_from}, {ref_to, port_to}})
-            table.insert(ref_from.next_layers, ref_to) -- add edge
-            ref_to.in_deg = ref_to.in_deg + 1          -- increase the in-degree of the target layer
-        end
-    end
-
-    -- topology sort
-    local queue = {}
-    local l = 1
-    local r = 1
-    for id, ref in pairs(layers) do
-        if ref.in_deg == 0 then
-            table.insert(queue, ref)
-            nerv.info("adding source layer: %s", id)
-            r = r + 1
-        end
-    end
-    if l == r then
-        nerv.error("loop detected")
-    end
-    while l < r do
-        local cur = queue[l]
-        cur.visited = true
-        l = l + 1
-        for _, nl in pairs(cur.next_layers) do
-            nl.in_deg = nl.in_deg - 1
-            if nl.in_deg == 0 then
-                table.insert(queue, nl)
-                r = r + 1
-            end
-        end
-    end
-    for i = 1, #queue do
-        nerv.info("enqueued layer: %s %s", queue[i].layer, queue[i].layer.id)
-    end
-
-    for id, ref in pairs(layers) do
-        -- check wether the graph is connected
-        if ref.visited == false then
-            nerv.warning("layer %s is ignored", id)
-        end
-    end
-
-    nerv.Layer.__init(self, id, global_conf, layer_conf)
-    self.layers = layers
-    self.inputs = inputs
-    self.outputs = outputs
-    self.parsed_conn = parsed_conn
-    self.queue = queue
-end
-
-function DAGLayer:bind_params()
-    -- do nothing (instead of rebinding params for each layer)
-end
-
-function DAGLayer:init(batch_size, chunk_size)
-    if chunk_size == nil then
-        chunk_size = 1
-    end
-    for i, conn in ipairs(self.parsed_conn) do
-        local _, output_dim
-        local ref_from, port_from, ref_to, port_to
-        ref_from, port_from = unpack(conn[1])
-        ref_to, port_to = unpack(conn[2])
-        _, output_dim = ref_from.layer:get_dim()
-        local dim = 1
-        if output_dim[port_from] > 0 then
-            dim = output_dim[port_from]
-        end
-
-        for t = 1, chunk_size do
-            local mid = self.mat_type(batch_size, dim)
-            local err_mid = mid:create()
-            touch_list_by_idx(ref_to.inputs, t)
-            touch_list_by_idx(ref_from.outputs, t)
-            touch_list_by_idx(ref_from.err_inputs, t)
-            touch_list_by_idx(ref_to.err_outputs, t)
-
-            ref_from.outputs[t][port_from] = mid
-            ref_to.inputs[t][port_to] = mid
-
-            ref_from.err_inputs[t][port_from] = err_mid
-            ref_to.err_outputs[t][port_to] = err_mid
-        end
-    end
-    for id, ref in pairs(self.layers) do
-        for i = 1, ref.input_len do
-            if ref.inputs[1][i] == nil then
-                nerv.error("dangling input port %d of layer %s", i, id)
-            end
-        end
-        for i = 1, ref.output_len do
-            if ref.outputs[1][i] == nil then
-                nerv.error("dangling output port %d of layer %s", i, id)
-            end
-        end
-        -- initialize sub layers
-        ref.layer:init(batch_size, chunk_size)
-    end
-    for i = 1, #self.dim_in do
-        if self.inputs[i] == nil then
-            nerv.error("dangling port %d of layer <input>", i)
-        end
-    end
-    for i = 1, #self.dim_out do
-        if self.outputs[i] == nil then
-            nerv.error("dangling port %d of layer <output>", i)
-        end
-    end
-end
-
-function DAGLayer:batch_resize(batch_size, chunk_size)
-    if chunk_size == nil then
-        chunk_size = 1
-    end
-
-    for i, conn in ipairs(self.parsed_conn) do
-        local _, output_dim
-        local ref_from, port_from, ref_to, port_to
-        ref_from, port_from = unpack(conn[1])
-        ref_to, port_to = unpack(conn[2])
-        _, output_dim = ref_from.layer:get_dim()
-
-        if ref_from.outputs[1][port_from]:nrow() ~= batch_size
-            and output_dim[port_from] > 0 then
-            for t = 1, chunk_size do
-                local mid = self.mat_type(batch_size, output_dim[port_from])
-                local err_mid = mid:create()
-
-                ref_from.outputs[t][port_from] = mid
-                ref_to.inputs[t][port_to] = mid
-
-                ref_from.err_inputs[t][port_from] = err_mid
-                ref_to.err_outputs[t][port_to] = err_mid
-            end
-        end
-    end
-    for id, ref in pairs(self.layers) do
-        ref.layer:batch_resize(batch_size, chunk_size)
-    end
-    collectgarbage("collect")
-end
-
-function DAGLayer:set_inputs(input, t)
-    for i = 1, #self.dim_in do
-        if input[i] == nil then
-            nerv.error("some input is not provided");
-        end
-        local layer = self.inputs[i][1]
-        local port = self.inputs[i][2]
-        touch_list_by_idx(layer.inputs, t)
-        layer.inputs[t][port] = input[i]
-    end
-end
-
-function DAGLayer:set_outputs(output, t)
-    for i = 1, #self.dim_out do
-        if output[i] == nil then
-            nerv.error("some output is not provided");
-        end
-        local layer = self.outputs[i][1]
-        local port = self.outputs[i][2]
-        touch_list_by_idx(layer.outputs, t)
-        layer.outputs[t][port] = output[i]
-    end
-end
-
-function DAGLayer:set_err_inputs(bp_err, t)
-    for i = 1, #self.dim_out do
-        local layer = self.outputs[i][1]
-        local port = self.outputs[i][2]
-        touch_list_by_idx(layer.err_inputs, t)
-        layer.err_inputs[t][port] = bp_err[i]
-    end
-end
-
-function DAGLayer:set_err_outputs(next_bp_err, t)
-    for i = 1, #self.dim_in do
-        local layer = self.inputs[i][1]
-        local port = self.inputs[i][2]
-        touch_list_by_idx(layer.err_outputs, t)
-        layer.err_outputs[t][port] = next_bp_err[i]
-    end
-end
-
-function DAGLayer:update(bp_err, input, output, t)
-    if t == nil then
-        t = 1
-    end
-    self:set_err_inputs(bp_err, t)
-    self:set_inputs(input, t)
-    self:set_outputs(output, t)
-    for id, ref in pairs(self.queue) do
-        ref.layer:update(ref.err_inputs[t], ref.inputs[t], ref.outputs[t], t)
-    end
-end
-
-function DAGLayer:propagate(input, output, t)
-    if t == nil then
-        t = 1
-    end
-    self:set_inputs(input, t)
-    self:set_outputs(output, t)
-    local ret = false
-    for i = 1, #self.queue do
-        local ref = self.queue[i]
-        ret = ref.layer:propagate(ref.inputs[t], ref.outputs[t], t)
-    end
-    return ret
-end
-
-function DAGLayer:back_propagate(bp_err, next_bp_err, input, output, t)
-    if t == nil then
-        t = 1
-    end
-    self:set_err_outputs(next_bp_err, t)
-    self:set_err_inputs(bp_err, t)
-    self:set_inputs(input, t)
-    self:set_outputs(output, t)
-    for i = #self.queue, 1, -1 do
-        local ref = self.queue[i]
-        ref.layer:back_propagate(ref.err_inputs[t], ref.err_outputs[t], ref.inputs[t], ref.outputs[t], t)
-    end
-end
-
-function DAGLayer:get_params()
-    local param_repos = {}
-    for id, ref in pairs(self.queue) do
-        table.insert(param_repos, ref.layer:get_params())
-    end
-    return nerv.ParamRepo.merge(param_repos, self.loc_type)
-end
-
-DAGLayer.PORT_TYPES = {
-    INPUT = {},
-    OUTPUT = {},
-    ERR_INPUT = {},
-    ERR_OUTPUT = {}
-}
-
-function DAGLayer:get_intermediate(id, port_type)
-    if id == "<input>" or id == "<output>" then
-        nerv.error("an actual real layer id is expected")
-    end
-    local layer = self.layers[id]
-    if layer == nil then
-        nerv.error("layer id %s not found", id)
-    end
-    if port_type == DAGLayer.PORT_TYPES.INPUT then
-        return layer.inputs
-    elseif port_type == DAGLayer.PORT_TYPES.OUTPUT then
-        return layer.outputs
-    elseif port_type == DAGLayer.PORT_TYPES.ERR_INPUT then
-        return layer.err_inputs
-    elseif port_type == DAGLayer.PORT_TYPES.ERR_OUTPUT then
-        return layer.err_outputs
-    end
-    nerv.error("unrecognized port type")
-end
diff --git a/nerv/nn/network.lua b/nerv/nn/network.lua
new file mode 100644
index 0000000..2cb83ce
--- /dev/null
+++ b/nerv/nn/network.lua
@@ -0,0 +1,500 @@
+local network = nerv.class('nerv.Network')
+
+function network:__init(id, global_conf, network_conf)
+    self.id = id
+    self.network = network_conf.network
+    self.dim_in = self.network.dim_in
+    self.dim_out = self.network.dim_out
+    self.gconf = global_conf
+    if self.gconf.use_cpu then
+        self.mat_type = self.gconf.mmat_type
+    else
+        self.mat_type = self.gconf.cumat_type
+    end
+    self.clip = network_conf.clip
+    self.nn_act_default = network_conf.nn_act_default
+    if self.nn_act_default == nil then
+        self.nn_act_default = 0
+    end
+    self.layers = {}
+    self.input_conn = {}
+    self.output_conn = {}
+    self.socket = self:compile(self.network)
+    for i = 1, #self.dim_in do
+        local edge = self.socket.inputs[i]
+        local id, port, time = edge[1], edge[2], edge[3]
+        if self.input_conn[id][port] ~= nil then
+            nerv.error('duplicate edge')
+        end
+        self.input_conn[id][port] = {0, i, time}
+    end
+    for i = 1, #self.dim_out do
+        local edge = self.socket.outputs[i]
+        local id, port, time = edge[1], edge[2], edge[3]
+        if self.output_conn[id][port] ~= nil then
+            nerv.error('duplicate edge')
+        end
+        self.output_conn[id][port] = {0, i, time}
+    end
+    self.delay = 0
+    for i = 1, #self.layers do
+        local dim_in, _ = self.layers[i]:get_dim()
+        for j = 1, #dim_in do
+            local time = self.input_conn[i][j][3]
+            if math.abs(time) > self.delay then
+                self.delay = math.abs(time)
+            end
+        end
+    end
+end
+
+function network:compile(layer)
+    local socket = {inputs = {}, outputs = {}}
+    if not nerv.is_type(layer, 'nerv.GraphLayer') then
+        table.insert(self.layers, layer)
+        local id = #self.layers
+        self.input_conn[id] = {}
+        self.output_conn[id] = {}
+        local dim_in, dim_out = layer:get_dim()
+        for i = 1, #dim_in do
+            socket.inputs[i] = {id, i, 0}
+        end
+        for i = 1, #dim_out do
+            socket.outputs[i] = {id, i, 0}
+        end
+    else
+        local sublayer_socket = {}
+        for id, sublayer in pairs(layer.layers) do
+            if id ~= '<input>' then
+               sublayer_socket[sublayer.id] = self:compile(sublayer.layer)
+            end
+        end
+        for _, edge in pairs(layer.connections) do
+            -- id = 0 means <input> or <output>
+            local id_from, port_from = edge[1], edge[2]
+            local id_to, port_to = edge[3], edge[4]
+            local time = edge[5]
+            if id_from == 0 then
+                if socket.inputs[port_from] ~= nil then
+                    nerv.error('duplicate input socket')
+                end
+                local input = sublayer_socket[id_to].inputs[port_to]
+                local id, port, t = input[1], input[2], input[3] + time
+                socket.inputs[port_from] = {id, port, t}
+            else
+                local output = sublayer_socket[id_from].outputs[port_from]
+                local id, port, t = output[1], output[2], output[3] + time
+                if id_to == 0 then
+                    if socket.outputs[port_to] ~= nil then
+                        nerv.error('duplicate output socket')
+                    end
+                    socket.outputs[port_to] = {id, port, t}
+                else
+                    local input = sublayer_socket[id_to].inputs[port_to]
+                    local id1, port1, t1 = input[1], input[2], input[3]
+                    if self.input_conn[id1][port1] ~= nil or self.output_conn[id][port] ~= nil then
+                        nerv.error('duplicate edge')
+                    end
+                    self.input_conn[id1][port1] = {id, port, t + t1}
+                    self.output_conn[id][port] = {id1, port1, t + t1}
+                end
+            end
+        end
+    end
+    return socket
+end
+
+function network:init(batch_size, chunk_size)
+    self.batch_size = batch_size
+    self.chunk_size = chunk_size
+
+    self:topsort()
+
+    self:make_initial_store()
+    collectgarbage('collect')
+end
+
+function network:epoch_init()
+    for i = 1, #self.layers do
+        self.layers[i]:init(self.batch_size, self.chunk_size)
+    end
+end
+
+function network:topsort()
+    nerv.info('network topology sort')
+    local degree = {}
+    for t = 1, self.chunk_size do
+        degree[t] = {}
+        for i = 1, #self.layers do
+            degree[t][i] = 0
+        end
+    end
+
+    for t = 1, self.chunk_size do
+        for i = 1, #self.layers do
+            local _, dim_out = self.layers[i]:get_dim()
+            for j = 1, #dim_out do
+                if self.output_conn[i][j] ~= nil then
+                    local edge = self.output_conn[i][j]
+                    local id, time = edge[1], edge[3] + t
+                    if time >= 1 and time <= self.chunk_size and id ~= 0 then
+                        degree[time][id] = degree[time][id] + 1
+                    end
+                end
+            end
+        end
+    end
+
+    self.queue = {}
+    local l = 1
+    local r = 0
+    for t = 1, self.chunk_size do
+        for i = 1, #self.layers do
+            if degree[t][i] == 0 then
+                r = r + 1
+                self.queue[r] = {chunk = t, id = i}
+            end
+        end
+    end
+    while l <= r do
+        local t, i = self.queue[l].chunk, self.queue[l].id
+        l = l + 1
+        local _, dim_out = self.layers[i]:get_dim()
+        for j = 1, #dim_out do
+            if self.output_conn[i][j] ~= nil then
+                local edge = self.output_conn[i][j]
+                local id, time = edge[1], edge[3] + t
+                if time >= 1 and time <= self.chunk_size and id ~= 0 then
+                    degree[time][id] = degree[time][id] - 1
+                    if degree[time][id] == 0 then
+                        r = r + 1
+                        self.queue[r] = {chunk = time, id = id}
+                    end
+                end
+            end
+        end
+    end
+
+    if r ~= self.chunk_size * #self.layers then
+        nerv.error('loop detected')
+    end
+end
+
+function network:make_initial_store()
+    nerv.info('network initing storage')
+
+    -- allocate memory
+    local memory = {}
+    local err_memory = {}
+    for t = 1 - self.delay, self.chunk_size + self.delay do
+        memory[t] = {}
+        err_memory[t] = {}
+        for i = 1, #self.layers do
+            memory[t][i] = {}
+            err_memory[t][i] = {}
+            local dim_in, dim_out = self.layers[i]:get_dim()
+            for j = 1, #dim_in do
+                err_memory[t][i][j] = self.mat_type(self.batch_size, dim_in[j])
+                err_memory[t][i][j]:fill(0)
+            end
+            for j = 1, #dim_out do
+                memory[t][i][j] = self.mat_type(self.batch_size, dim_out[j])
+                memory[t][i][j]:fill(self.nn_act_default)
+            end
+        end
+        -- memory[t][0] stores network input
+        memory[t][0] = {}
+        for j = 1, #self.dim_in do
+            memory[t][0][j] = self.mat_type(self.batch_size, self.dim_in[j])
+            memory[t][0][j]:fill(self.nn_act_default)
+        end
+        -- err_memory[t][0] stores network err_input
+        err_memory[t][0] = {}
+        for j = 1, #self.dim_out do
+            err_memory[t][0][j] = self.mat_type(self.batch_size, self.dim_out[j])
+            err_memory[t][0][j]:fill(0)
+        end
+    end
+
+    -- connect memory and reference
+    self.input = {}
+    self.output = {}
+    self.err_input = {}
+    self.err_output = {}
+    for t = 1, self.chunk_size do
+        self.input[t] = {}
+        self.output[t] = {}
+        self.err_input[t] = {}
+        self.err_output[t] = {}
+        for i = 1, #self.layers do
+            self.input[t][i] = {}
+            self.output[t][i] = {}
+            self.err_input[t][i] = {}
+            self.err_output[t][i] = {}
+            local dim_in, dim_out = self.layers[i]:get_dim()
+            for j = 1, #dim_in do
+                local edge = self.input_conn[i][j]
+                local id, port, time = edge[1], edge[2], edge[3]
+                if id ~= 0 or t - time < 1 or t - time > self.chunk_size then
+                    self.input[t][i][j] = memory[t - time][id][port]
+                end
+                if id ~= 0 then
+                    self.err_output[t][i][j] = err_memory[t][i][j]
+                end
+            end
+            for j = 1, #dim_out do
+                local edge = self.output_conn[i][j]
+                local id, port, time = edge[1], edge[2], edge[3]
+                if id ~= 0 then
+                    self.output[t][i][j] = memory[t][i][j]
+                end
+                if id ~= 0 or t + time < 1 or t + time > self.chunk_size then
+                    self.err_input[t][i][j] = err_memory[t + time][id][port]
+                end
+            end
+        end
+    end
+
+    -- check dangling reference
+    for t = 1, self.chunk_size do
+        for i = 1, #self.dim_in do
+            local edge = self.socket.inputs[i]
+            local id, port, time = edge[1], edge[2], edge[3]
+            if t + time >= 1 and t + time <= self.chunk_size then
+                if self.input[t + time][id][port] ~= nil then
+                    nerv.error('input reference not nil')
+                end
+                self.input[t + time][id][port] = true      -- just a place holder
+                if self.err_output[t + time][id][port] ~= nil then
+                    nerv.error('err_output reference not nil')
+                end
+                self.err_output[t + time][id][port] = true -- just a place holder
+            end
+        end
+        for i = 1, #self.dim_out do
+            local edge = self.socket.outputs[i]
+            local id, port, time = edge[1], edge[2], edge[3]
+            if t - time >= 1 and t - time <= self.chunk_size then
+                if self.output[t - time][id][port] ~= nil then
+                    nerv.error('output reference not nil')
+                end
+                self.output[t - time][id][port] = true     -- just a place holder
+                if self.err_input[t - time][id][port] ~= nil then
+                    nerv.error('err_output reference not nil')
+                end
+                self.err_input[t - time][id][port] = true  -- just a place holder
+            end
+        end
+    end
+    for t = 1, self.chunk_size do
+        for i = 1, #self.layers do
+            local dim_in, dim_out = self.layers[i]:get_dim()
+            for j = 1, #dim_in do
+                if self.input[t][i][j] == nil then
+                    nerv.error('input reference dangling')
+                end
+                if self.err_output[t][i][j] == nil then
+                    nerv.error('err_output reference dangling')
+                end
+            end
+            for j = 1, #dim_out do
+                if self.output[t][i][j] == nil then
+                    nerv.error('output reference dangling')
+                end
+                if self.err_input[t][i][j] == nil then
+                    nerv.error('err_input reference dangling')
+                end
+            end
+        end
+    end
+
+    -- allocate reference for legacy of previous mini-batch
+    self.legacy = {}
+    for t = 1 - self.delay, 0 do
+        self.legacy[t] = {}
+        for i = 1, #self.layers do
+            self.legacy[t][i] = {}
+            local _, dim_out = self.layers[i]:get_dim()
+            for j = 1, #dim_out do
+                self.legacy[t][i][j] = memory[t][i][j]
+            end
+        end
+    end
+end
+
+function network:set_input(input)
+    for t = 1, self.chunk_size do
+        for i = 1, #self.dim_in do
+            local edge = self.socket.inputs[i]
+            local id, port, time = edge[1], edge[2], edge[3]
+            if t + time >= 1 and t + time <= self.chunk_size then
+                self.input[t + time][id][port] = input[t][i]
+            end
+        end
+    end
+end
+
+function network:set_output(output)
+    for t = 1, self.chunk_size do
+        for i = 1, #self.dim_out do
+            local edge = self.socket.outputs[i]
+            local id, port, time = edge[1], edge[2], edge[3]
+            if t - time >= 1 and t - time <= self.chunk_size then
+                self.output[t - time][id][port] = output[t][i]
+            end
+        end
+    end
+end
+
+function network:set_err_input(err_input)
+    for t = 1, self.chunk_size do
+        for i = 1, #self.dim_out do
+            local edge = self.socket.outputs[i]
+            local id, port, time = edge[1], edge[2], edge[3]
+            if t - time >= 1 and t - time <= self.chunk_size then
+                self.err_input[t - time][id][port] = err_input[t][i]
+            end
+        end
+    end
+end
+
+function network:set_err_output(err_output)
+    for t = 1, self.chunk_size do
+        for i = 1, #self.dim_in do
+            local edge = self.socket.inputs[i]
+            local id, port, time = edge[1], edge[2], edge[3]
+            if t + time >= 1 and t + time <= self.chunk_size then
+                self.err_output[t + time][id][port] = err_output[t][i]
+            end
+        end
+    end
+end
+
+--[[
+    [info] is a table that contains information of current mini-batch. These fields must be contained:
+        [input], [output] : matrix array which stores the network input and output
+        [seq_length] : a table contains the length of every sequences
+        [new_seq]: a table contains the batch number of new sequences
+        [do_train]: a bool value indicates do train or not
+        if [do_train] is true, these fileds also must be contained:
+            [err_input], [err_output] : matrix array which stores the network err_input and err_output
+--]]
+function network:mini_batch_init(info)
+    self.info = info
+    self:set_input(self.info.input)
+    self:set_output(self.info.output)
+
+    -- calculate border
+    self.max_length = 0
+    self.border = {}
+    for i = 1, self.chunk_size do
+        self.border[i] = {}
+    end
+    for i = 1, self.batch_size do
+        if self.info.seq_length[i] > self.max_length then
+            self.max_length = self.info.seq_length[i]
+        end
+        for t = 1, self.delay do
+            local chunk = self.info.seq_length[i] + t
+            if chunk > self.chunk_size then
+                break
+            end
+            table.insert(self.border[chunk], i)
+        end
+    end
+
+    -- copy legacy
+    for t = 1 - self.delay, 0 do
+        for i = 1, #self.layers do
+            local _, dim_out = self.layers[i]:get_dim()
+            for j = 1, #dim_out do
+                if t + self.chunk_size >= 1 and self.output_conn[i][j][1] ~= 0 then
+                    self.legacy[t][i][j]:copy_from(self.output[t + self.chunk_size][i][j])
+                end
+                for k = 1, #self.info.new_seq do
+                    local batch = self.info.new_seq[k]
+                    self.legacy[t][i][j][batch - 1]:fill(self.nn_act_default)
+                end
+            end
+        end
+    end
+
+    if self.info.do_train then
+        self:set_err_input(self.info.err_input)
+        self:set_err_output(self.info.err_output)
+
+        -- flush border gradient
+        for t = self.max_length + 1, self.max_length + self.delay do
+            if t > self.chunk_size then
+                break
+            end
+            for i = 1, #self.layers do
+                local dim_in, _ = self.layers[i]:get_dim()
+                for j = 1, #dim_in do
+                    self.err_output[t][i][j]:fill(0)
+                end
+            end
+        end
+    end
+end
+
+function network:propagate()
+    for i = 1, #self.queue do
+        local t, id = self.queue[i].chunk, self.queue[i].id
+        if t <= self.max_length then
+            self.layers[id]:propagate(self.input[t][id], self.output[t][id], t)
+        end
+        -- flush border activation
+        for j = 1, #self.border[t] do
+            local batch = self.border[t][j]
+            local _, dim_out = self.layers[id]:get_dim()
+            for k = 1, #dim_out do
+                self.output[t][id][k][batch - 1]:fill(self.nn_act_default)
+            end
+        end
+    end
+end
+
+function network:back_propagate()
+    for i = #self.queue, 1, -1 do
+        local t, id = self.queue[i].chunk, self.queue[i].id
+        if t <= self.max_length then
+            -- flush border gradient
+            for j = 1, #self.border[t] do
+                local batch = self.border[t][j]
+                local _, dim_out = self.layers[id]:get_dim()
+                for k = 1, #dim_out do
+                    self.err_input[t][id][k][batch - 1]:fill(0)
+                end
+            end
+            self.layers[id]:back_propagate(self.err_input[t][id], self.err_output[t][id], self.input[t][id], self.output[t][id], t)
+            if self.clip ~= nil then
+                local dim_in, _ = self.layers[id]:get_dim()
+                for j = 1, #dim_in do
+                    self.err_output[t][id][j]:clip(-self.clip, self.clip)
+                end
+            end
+        end
+    end
+end
+
+function network:update()
+    for i = 1, #self.queue do
+        local t, id = self.queue[i].chunk, self.queue[i].id
+        if t <= self.max_length then
+            self.layers[id]:update(self.err_input[t][id], self.input[t][id], self.output[t][id], t)
+        end
+    end
+end
+
+function network:set_attr(name, value)
+    self.network:set_attr(name, value)
+end
+
+function network:get_sublayer(id)
+    return self.network:get_sublayer(id)
+end
+
+function network:get_params()
+    return self.network:get_params()
+end
diff --git a/nerv/tnn/init.lua b/nerv/tnn/init.lua
deleted file mode 100644
index 44ce26b..0000000
--- a/nerv/tnn/init.lua
+++ /dev/null
@@ -1,47 +0,0 @@
-local LayerT = nerv.class('nerv.LayerT')
-
-function LayerT:__init(id, global_conf, layer_conf)
-    nerv.error_method_not_implemented()
-end
-
-function LayerT:init(batch_size, chunk_size)
-    nerv.error_method_not_implemented()
-end
-
-function LayerT:update(bp_err, input, output, t)
-    nerv.error_method_not_implemented()
-end
-
-function LayerT:propagate(input, output, t)
-    nerv.error_method_not_implemented()
-end
-
-function LayerT:back_propagate(bp_err, next_bp_err, input, output, t)
-    nerv.error_method_not_implemented()
-end
-
-function LayerT:check_dim_len(len_in, len_out)
-    local expected_in = #self.dim_in
-    local expected_out = #self.dim_out
-    if len_in > 0 and expected_in ~= len_in then
-        nerv.error("layer %s expects %d inputs, %d given",
-                    self.id, len_in, expected_in)
-    end
-    if len_out > 0 and expected_out ~= len_out then
-        nerv.error("layer %s expects %d outputs, %d given",
-                    self.id, len_out, expected_out)
-    end
-end
-
-LayerT.find_param = nerv.Layer.find_param
-
-function LayerT:get_params()
-    nerv.error_method_not_implemented()
-end
-
-function LayerT:get_dim()
-    return self.dim_in, self.dim_out
-end
-
-nerv.include('sutil.lua')
-nerv.include('tnn.lua')
diff --git a/nerv/tnn/sutil.lua b/nerv/tnn/sutil.lua
deleted file mode 100644
index 6a968b7..0000000
--- a/nerv/tnn/sutil.lua
+++ /dev/null
@@ -1,80 +0,0 @@
-local Util = nerv.class("nerv.SUtil") --Scheduler Utility
-
-function Util.simple_split(inputstr, sep)
-    if sep == nil then
-        sep = "%s"
-    end
-    local t={} ; i=1
-    for str in string.gmatch(inputstr, "([^"..sep.."]+)") do
-        t[i] = str
-        i = i + 1
-    end
-    return t
-end
-
-function Util.parse_schedule(str)
-    --parse a string like "1.2*10:1.5" to a list of numbers
-    local sch = {}
-    local s = Util.simple_split(str, ':')
-    for i = 1, #s do
-        local p = Util.simple_split(s[i], "%*")
-        if #p ~= 2 and #p ~= 1 then
-            nerv.error("nerv.SUtil:parse_schedule error, unit(%s) not proper, has %d components.", s[i], #p)
-        end
-        if p[2] == nil then
-            p[2] = "1"
-        end 
-        p[1] = tonumber(p[1])
-        p[2] = tonumber(p[2])
-        for j = 1, p[2] do
-            table.insert(sch, p[1])
-        end 
-    end 
-
-    --for i = 1, #sch do
-    --    print(sch[i])
-    --end
-    return sch
-end
-
-function Util.sche_get(s, it)
-    --get s[it]
-    if s == nil then
-        nerv.info("Util.sche_get: warning, scheule is nil, returning zero...")
-        return 0
-    end
-    if #s >= it then
-        return s[it]
-    else
-        nerv.info("Util.sche_get: warning, it(%d) > #schedule(%d), returning the last one of schedule(%f)...", it, #s, s[#s])
-        return s[#s]
-    end
-end
-
-function Util.parse_commands_set(str)
-    local coms = {}
-    local s = Util.simple_split(str, ':,')
-    for i = 1 ,#s do
-        if coms[s[i]] == 1 then
-            nerv.warning("nerv.SUtil.parse_commands_set command(%s) appered more than once in command_set(%s)", s[i], str)
-        end
-        coms[s[i]] = 1
-    end
-    return coms
-end
-
-function Util.log_redirect(fn)
-    nerv.log_fh = assert(io.open(fn, "w"))
-    nerv.info("CAUTION[LOG_REDIRECT], all nerv.printf/info/warning/error calls will be double-written to %s", fn)
-    nerv.printf = 
-        function (fmt, ...)
-            io.write(nerv.sprintf(fmt, ...))
-            nerv.log_fh:write(nerv.sprintf(fmt, ...))
-            nerv.log_fh:flush()
-        end
-    nerv.error = 
-        function (fmt, ...)
-            nerv.log_fh:write(nerv.sprintf("[nerv] internal error:" .. fmt .. "\n", ...))
-            error(nerv.sprintf("[nerv] internal error: " .. fmt .. "\n", ...))
-        end
-end
diff --git a/nerv/tnn/tnn.lua b/nerv/tnn/tnn.lua
deleted file mode 100644
index d527fe6..0000000
--- a/nerv/tnn/tnn.lua
+++ /dev/null
@@ -1,596 +0,0 @@
-local TNN = nerv.class("nerv.TNN")
-
-local function parse_id(str)
-    --used to parse layerid[portid],time
-    local id, port, time, _
-    _, _, id, port, time = string.find(str, "([a-zA-Z0-9_]+)%[([0-9]+)%][,]*([0-9]*)")
-    if id == nil or port == nil then
-        _, _, id, port, time = string.find(str, "(.+)%[([0-9]+)%][,]*([0-9]*)")
-        if not (id == "<input>" or id == "<output>") then
-            nerv.error("wrong format of connection id")
-        end
-    end
-    --print(str, id, port, time)
-    port = tonumber(port)
-    if (time == nil) then
-        time = 0
-    else
-        time = tonumber(time)
-    end
-    --now time don't need to be parsed
-    return id, port
-end
-
-local function discover(id, layers, layer_repo)
-    local ref = layers[id]
-    if id == "<input>" or id == "<output>" then
-        return nil
-    end
-    if ref == nil then
-        local layer = layer_repo:get_layer(id)
-        local dim_in, dim_out = layer:get_dim()
-        ref = {
-            layer = layer,
-            id = layer.id,
-            inputs_m = {}, --storage for computation, inputs_m[time][port]
-            inputs_b = {}, --inputs_g[time][port], whether this input can been computed
-            inputs_matbak_p = {}, --which is a back-up space to handle some cross-border computation, inputs_p_matbak[port]
-            outputs_m = {},
-            outputs_b = {},
-            err_inputs_m = {},
-            err_inputs_matbak_p = {}, --which is a back-up space to handle some cross-border computation
-            err_inputs_b = {},
-            err_outputs_m = {},
-            err_outputs_b = {},
-            i_conns_p = {}, --list of inputing connections
-            o_conns_p = {}, --list of outputing connections
-            dim_in = dim_in, --list of dimensions of ports
-            dim_out = dim_out,
-        }
-        layers[id] = ref
-    end
-    return ref
-end
-
-nerv.TNN.FC = {} --flag const
-nerv.TNN.FC.SEQ_START = 4
-nerv.TNN.FC.SEQ_END = 8 
-nerv.TNN.FC.HAS_INPUT = 1
-nerv.TNN.FC.HAS_LABEL = 2
-nerv.TNN.FC.SEQ_NORM = bit.bor(nerv.TNN.FC.HAS_INPUT, nerv.TNN.FC.HAS_LABEL) --This instance have both input and label
-
-function TNN.make_initial_store(st, p, dim, batch_size, chunk_size, extend_t, global_conf, st_c, p_c, t_c)
-    --Return a table of matrix storage from time (1-extend_t)..(chunk_size+extend_t)
-    if (type(st) ~= "table") then
-        nerv.error("st should be a table")
-    end
-    for i = 1 - extend_t - 2, chunk_size + extend_t + 2 do --intentionally allocated more time
-        if (st[i] == nil) then
-            st[i] = {}
-        end
-        st[i][p] = global_conf.cumat_type(batch_size, dim)
-        st[i][p]:fill(0)
-        if (st_c ~= nil) then
-            if (st_c[i + t_c] == nil) then
-                st_c[i + t_c] = {}
-            end
-            st_c[i + t_c][p_c] = st[i][p]
-        end
-    end
-    collectgarbage("collect") --free the old one to save memory
-end
-
-function TNN:out_of_feedrange(t) --out of chunk, or no input, for the current feed
-    if (t < 1 or t > self.chunk_size) then
-        return true
-    end
-    if (self.feeds_now.flagsPack_now[t] == 0 or self.feeds_now.flagsPack_now[t] == nil) then
-        return true
-    end
-    return false
-end
-
-function TNN:__init(id, global_conf, layer_conf)
-    self.clip_t = layer_conf.clip_t
-    if self.clip_t == nil then
-        self.clip_t = 0
-    end
-    if self.clip_t > 0 then
-        nerv.info("tnn(%s) will clip gradient across time with %f...", id, self.clip_t)
-    end
-
-    self.extend_t = layer_conf.extend_t --TNN will allocate storage of time for 1-extend_t .. chunk_size+extend_t
-    if self.extend_t == nil then
-        self.extend_t = 5 
-    end
-    nerv.info("tnn(%s) will extend storage beyond MB border for time steps %d...", id, self.extend_t)
-
-    local layers = {}
-    local inputs_p = {} --map:port of the TNN to layer ref and port
-    local outputs_p = {}
-    local dim_in = layer_conf.dim_in
-    local dim_out = layer_conf.dim_out
-    local parsed_conns = {}
-    local _
-    
-    for id, _ in pairs(layer_conf.sub_layers.layers) do --caution: with this line, some layer not connected will be included
-        discover(id, layers, layer_conf.sub_layers)
-    end
-
-    for _, ll in pairs(layer_conf.connections) do
-        local id_from, port_from = parse_id(ll[1])
-        local id_to, port_to = parse_id(ll[2])
-        local time_to = ll[3] 
-
-        print(id_from, id_to, time_to)
-
-        local ref_from = discover(id_from, layers, layer_conf.sub_layers)
-        local ref_to = discover(id_to, layers, layer_conf.sub_layers)
-        
-        if (id_from == "<input>") then
-            if (dim_in[port_from] ~= ref_to.dim_in[port_to] or time_to ~= 0) then
-                nerv.error("mismatch dimension or wrong time %s,%s,%d", ll[1], ll[2], ll[3])
-            end
-            inputs_p[port_from] = {["ref"] = ref_to, ["port"] = port_to}          
-            ref_to.inputs_m[port_to] = {} --just a place holder
-        elseif (id_to == "<output>") then
-            if (dim_out[port_to] ~= ref_from.dim_out[port_from] or time_to ~= 0) then
-                nerv.error("mismatch dimension or wrong time %s,%s,%d", ll[1], ll[2], ll[3])
-            end
-            outputs_p[port_to] = {["ref"] = ref_from, ["port"] = port_from}
-            ref_from.outputs_m[port_from] = {} --just a place holder
-        else
-            local conn_now = { 
-                ["src"] = {["ref"] = ref_from, ["port"] = port_from},
-                ["dst"] = {["ref"] = ref_to, ["port"] = port_to},
-                ["time"] = time_to
-            }
-            if (ref_to.dim_in[port_to] ~= ref_from.dim_out[port_from]) then
-                nerv.error("mismatch dimension or wrong time %s,%s,%d", ll[1], ll[2], ll[3])
-            end
-            table.insert(parsed_conns, conn_now)
-            ref_to.i_conns_p[conn_now.dst.port] = conn_now
-            ref_from.o_conns_p[conn_now.src.port] = conn_now
-        end
-    end
-
-    for id, ref in pairs(layers) do
-        print(id, "#dim_in:", #ref.dim_in, "#dim_out:", #ref.dim_out, "#i_conns_p:", #ref.i_conns_p, "#o_conns_p", #ref.o_conns_p)
-    end
-
-    self.layers = layers
-    self.inputs_p = inputs_p
-    self.outputs_p = outputs_p
-    self.id = id
-    self.dim_in = dim_in
-    self.dim_out = dim_out
-    self.parsed_conns = parsed_conns
-    self.gconf = global_conf
-end
-
-function TNN:init(batch_size, chunk_size)
-    self.batch_size = batch_size
-    self.chunk_size = chunk_size
-    for i, conn in ipairs(self.parsed_conns) do --init storage for connections inside the NN
-        local _, output_dim
-        local ref_from, port_from, ref_to, port_to, time
-        ref_from, port_from = conn.src.ref, conn.src.port
-        ref_to, port_to = conn.dst.ref, conn.dst.port
-        time = conn.time
-
-        local dim = ref_from.dim_out[port_from]
-        if (dim == 0) then
-            nerv.error("layer %s has a zero dim port", ref_from.layer.id)
-        end
-
-        nerv.info("TNN initing storage %s->%s", ref_from.layer.id, ref_to.layer.id)
-        ref_to.inputs_matbak_p[port_to] = self.gconf.cumat_type(batch_size, dim)
-        self.make_initial_store(ref_from.outputs_m, port_from, dim, batch_size, chunk_size, self.extend_t, self.gconf, ref_to.inputs_m, port_to, time)
-        ref_from.err_inputs_matbak_p[port_from] =  self.gconf.cumat_type(batch_size, dim)
-        self.make_initial_store(ref_from.err_inputs_m, port_from, dim, batch_size, chunk_size, self.extend_t, self.gconf, ref_to.err_outputs_m, port_to, time)
-    end
-
-    self.outputs_m = {}
-    self.err_inputs_m = {}
-    for i = 1, #self.dim_out do --Init storage for output ports 
-        local ref = self.outputs_p[i].ref
-        local p = self.outputs_p[i].port
-        self.make_initial_store(ref.outputs_m, p, self.dim_out[i], batch_size, chunk_size, self.extend_t, self.gconf, self.outputs_m, i, 0)
-        self.make_initial_store(ref.err_inputs_m, p, self.dim_out[i], batch_size, chunk_size, self.extend_t, self.gconf, self.err_inputs_m, i, 0)
-    end 
-
-    self.inputs_m = {}
-    self.err_outputs_m = {}
-    for i = 1, #self.dim_in do --Init storage for input ports 
-        local ref = self.inputs_p[i].ref
-        local p = self.inputs_p[i].port
-        self.make_initial_store(ref.inputs_m, p, self.dim_in[i], batch_size, chunk_size, self.extend_t, self.gconf, self.inputs_m, i, 0)
-        self.make_initial_store(ref.err_outputs_m, p, self.dim_in[i], batch_size, chunk_size, self.extend_t, self.gconf, self.err_outputs_m, i, 0)
-    end 
-
-    for id, ref in pairs(self.layers) do --Calling init for child layers
-        for i = 1, #ref.dim_in do
-            if (ref.inputs_m[i] == nil or ref.err_outputs_m[i] == nil) then
-                nerv.error("dangling input port %d of layer %s", i, id)
-            end
-        end
-        for i = 1, #ref.dim_out do
-            if (ref.outputs_m[i] == nil or ref.err_inputs_m[i] == nil) then
-                nerv.error("dangling output port %d of layer %s", i, id)
-            end
-        end
-        -- initialize sub layers
-        nerv.info("TNN initing sub-layer %s", ref.id)       
-        ref.layer:init(batch_size, chunk_size)
-        collectgarbage("collect")
-    end
-
-    local flags_now = {}
-    local flagsPack_now = {}
-    for i = 1, chunk_size do
-        flags_now[i] = {}
-        flagsPack_now[i] = 0
-    end
-
-    self.feeds_now = {} --feeds is for the reader to fill
-    self.feeds_now.inputs_m = self.inputs_m
-    self.feeds_now.flags_now = flags_now
-    self.feeds_now.flagsPack_now = flagsPack_now
-
-    self:flush_all()
-end
-
---[[
-function DAGLayer:batch_resize(batch_size)
-    self.gconf.batch_size = batch_size
-
-    for i, conn in ipairs(self.parsed_conn) do
-        local _, output_dim
-        local ref_from, port_from, ref_to, port_to
-        ref_from, port_from = unpack(conn[1])
-        ref_to, port_to = unpack(conn[2])
-        _, output_dim = ref_from.layer:get_dim()
-
-        if ref_from.outputs[port_from]:nrow() ~= batch_size and output_dim[port_from] > 0 then
-            local mid = self.gconf.cumat_type(batch_size, output_dim[port_from])
-            local err_mid = mid:create()
-
-            ref_from.outputs[port_from] = mid
-            ref_to.inputs[port_to] = mid
-
-            ref_from.err_inputs[port_from] = err_mid
-            ref_to.err_outputs[port_to] = err_mid
-        end
-    end
-    for id, ref in pairs(self.layers) do
-        ref.layer:batch_resize(batch_size)
-    end
-    collectgarbage("collect")
-end
-]]--
-
-function TNN:flush_all() --flush all history and activation
-    local _, ref
-    for _, ref in pairs(self.layers) do
-        for i = 1, #ref.dim_in do
-            for t = 1 - self.extend_t, self.chunk_size + self.extend_t do 
-                ref.inputs_m[t][i]:fill(self.gconf.nn_act_default)
-                if (ref.inputs_b[t] == nil) then
-                    ref.inputs_b[t] = {}
-                end
-                ref.inputs_b[t][i] = false
-                ref.err_outputs_m[t][i]:fill(0)
-                if (ref.err_outputs_b[t] == nil) then
-                    ref.err_outputs_b[t] = {}
-                end
-                ref.err_outputs_b[t][i] = false
-            end
-        end
-        for i = 1, #ref.dim_out do
-            for t = 1 - self.extend_t, self.chunk_size + self.extend_t do 
-                ref.outputs_m[t][i]:fill(self.gconf.nn_act_default)
-                if (ref.outputs_b[t] == nil) then
-                    ref.outputs_b[t] = {}
-                end
-                ref.outputs_b[t][i] = false
-                ref.err_inputs_m[t][i]:fill(0)
-                if (ref.err_inputs_b[t] == nil) then
-                    ref.err_inputs_b[t] = {}
-                end
-                ref.err_inputs_b[t][i] = false
-            end
-        end
-    end
-end
-
---reader: some reader
---Returns: bool, whether has new feed
---Returns: feeds, a table that will be filled with the reader's feeds
-function TNN:getfeed_from_reader(reader)
-    local feeds_now = self.feeds_now
-    local got_new = reader:get_batch(feeds_now)
-    return got_new, feeds_now
-end
-
-function TNN:move_right_to_nextmb(list_t) --move output history activations of 1..chunk_size to 1-chunk_size..0 
-    if list_t == nil then
-        list_t = {}
-        for i = self.extend_t, 1, -1 do
-            list_t[i] = 1 - i
-        end
-    end
-    for i = 1, #list_t do
-        t = list_t[i]
-        if t < 1 - self.extend_t or t > 0 then
-            nerv.error("MB move range error")
-        end
-        for id, ref in pairs(self.layers) do
-            for p = 1, #ref.dim_out do
-                ref.outputs_m[t][p]:copy_fromd(ref.outputs_m[t + self.chunk_size][p])
-            end
-        end
-    end
-end
-
-function TNN:net_propagate() --propagate according to feeds_now
-    for t = 1, self.chunk_size, 1 do
-        for id, ref in pairs(self.layers) do
-            for p = 1, #ref.dim_out do
-                ref.outputs_b[t][p] = false
-            end
-            for p = 1, #ref.dim_in do
-                ref.inputs_b[t][p] = false
-            end
-        end
-    end
-
-    local feeds_now = self.feeds_now
-    for t = 1, self.chunk_size do --some layer maybe do not have inputs from time 1..chunk_size
-        for id, ref in pairs(self.layers) do
-            if #ref.dim_in > 0 then --some layer is just there(only to save some parameter)
-                self:propagate_dfs(ref, t)
-            end
-        end
-    end
-    for t = 1, self.chunk_size do
-        if (bit.band(feeds_now.flagsPack_now[t], nerv.TNN.FC.HAS_INPUT) > 0) then
-            for i = 1, #self.dim_in do
-                local ref = self.inputs_p[i].ref
-                local p = self.inputs_p[i].port
-                ref.inputs_b[t][p] = true
-                self:propagate_dfs(ref, t)
-            end
-        end
-    end
-
-    local flag_out = true 
-    for t = 1, self.chunk_size do --check whether every output has been computed
-        if (bit.band(feeds_now.flagsPack_now[t], nerv.TNN.FC.HAS_LABEL) > 0) then
-            for i = 1, #self.dim_out do
-                local ref = self.outputs_p[i].ref
-                if (ref.outputs_b[t][1] ~= true) then
-                    flag_out = false 
-                    break
-                end
-            end
-        end
-    end
-
-    if (flag_out == false) then
-        nerv.error("some thing wrong, some labeled output is not propagated")
-    end
-end
-
---ref: the TNN_ref of a layer
---t: the current time to propagate
-function TNN:propagate_dfs(ref, t)
-    if (self:out_of_feedrange(t)) then
-        return
-    end
-    if (ref.outputs_b[t][1] == true) then --already propagated, 1 is just a random port
-        return
-    end
-
-    --print("debug dfs", ref.layer.id, t)
-
-    local flag = true --whether have all inputs
-    for _, conn in pairs(ref.i_conns_p) do
-        local p = conn.dst.port
-        if (not (ref.inputs_b[t][p] or self:out_of_feedrange(t - conn.time))) then
-            flag = false
-            break
-        end
-    end
-    if (flag == false) then
-        return
-    end
-
-    --ok, do propagate
-    --print("debug ok, propagating");
-    --The MB moving will cause bordering history to be changed, so it is more wise to flush the input activation
-    if (bit.band(self.feeds_now.flagsPack_now[t], bit.bor(nerv.TNN.FC.SEQ_START, nerv.TNN.FC.SEQ_END)) > 0) then --flush cross-border history
-        for i = 1, self.batch_size do
-            local seq_start = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_START) 
-            local seq_end = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_END) 
-            if (seq_start > 0 or seq_end > 0) then
-                for p, conn in pairs(ref.i_conns_p) do
-                    if ((ref.i_conns_p[p].time > 0 and seq_start > 0) or (ref.i_conns_p[p].time < 0 and seq_end > 0)) then --cross-border, set to default
-                        ref.inputs_m[t][p][i - 1]:fill(self.gconf.nn_act_default)
-                    end
-                end
-            end
-        end
-    end
-    self.gconf.timer:tic("tnn_actual_layer_propagate")
-    ref.layer:propagate(ref.inputs_m[t], ref.outputs_m[t], t) --propagate!
-    self.gconf.timer:toc("tnn_actual_layer_propagate")
-    --[[ 
-    if (bit.band(self.feeds_now.flagsPack_now[t], bit.bor(nerv.TNN.FC.SEQ_START, nerv.TNN.FC.SEQ_END)) > 0) then --restore cross-border history
-        for i = 1, self.batch_size do
-            local seq_start = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_START) 
-            local seq_end = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_END) 
-            if (seq_start > 0 or seq_end > 0) then
-                for p, conn in pairs(ref.o_conns_p) do
-                    if ((ref.o_conns_p[p].time > 0 and seq_end > 0) or (ref.o_conns_p[p].time < 0 and seq_start > 0)) then 
-                        ref.outputs_m[t][p][i - 1]:fill(self.gconf.nn_act_default)                       
-                    end
-                end
-            end
-        end
-    end
-    ]]--
-    --set input flag for future layers
-    for i = 1, #ref.dim_out do
-        if (ref.outputs_b[t][i] == true) then
-            nerv.error("this time's outputs_b should be false")
-        end
-        ref.outputs_b[t][i] = true
-    end
-    
-    --try dfs for further layers
-    for _, conn in pairs(ref.o_conns_p) do
-        --print("debug dfs-searching", conn.dst.ref.layer.id)
-        conn.dst.ref.inputs_b[t + conn.time][conn.dst.port] = true
-        self:propagate_dfs(conn.dst.ref, t + conn.time)
-    end
-end
-
---do_update: bool, whether we are doing back-propagate or updating the parameters
-function TNN:net_backpropagate(do_update) --propagate according to feeds_now
-    if do_update == nil then
-        nerv.error("do_update should not be nil")
-    end
-    for t = 1, self.chunk_size, 1 do
-        for id, ref in pairs(self.layers) do
-            for p = 1, #ref.dim_out do
-                ref.err_inputs_b[t][p] = false
-            end
-            for p = 1, #ref.dim_in do
-                ref.err_outputs_b[t][p] = false
-            end
-        end
-    end
-
-    local feeds_now = self.feeds_now
-    for t = 1, self.chunk_size do --some layer maybe do not have outputs from time 1..chunk_size
-        for id, ref in pairs(self.layers) do
-            if #ref.dim_out > 0 then --some layer is just there(only to save some parameter)
-                self:backpropagate_dfs(ref, t, do_update)
-            end
-        end
-    end
-    for t = 1, self.chunk_size do
-        if bit.band(feeds_now.flagsPack_now[t], nerv.TNN.FC.HAS_LABEL) > 0 then
-            for i = 1, #self.dim_out do
-                local ref = self.outputs_p[i].ref
-                local p = self.outputs_p[i].port
-                ref.err_inputs_b[t][p] = true
-                self:backpropagate_dfs(ref, t, do_update)
-            end
-        end
-    end
-
-    local flag_out = true 
-    for t = 1, self.chunk_size do --check whether every output has been computed
-        if bit.band(feeds_now.flagsPack_now[t], nerv.TNN.FC.HAS_INPUT) > 0 then
-            for i = 1, #self.dim_in do
-                local ref = self.inputs_p[i].ref
-                if ref.err_outputs_b[t][1] ~= true then
-                    flag_out = false 
-                    break
-                end
-            end
-        end
-    end
-    if (flag_out == false) then
-        nerv.error("some thing wrong, some input is not back_propagated")
-    end
-end
-
---ref: the TNN_ref of a layer
---t: the current time to propagate
-function TNN:backpropagate_dfs(ref, t, do_update)
-    if do_update == nil then
-        nerv.error("got a nil do_update")
-    end
-    if self:out_of_feedrange(t) then
-        return
-    end
-    if ref.err_outputs_b[t][1] == true then --already back_propagated, 1 is just a random port
-        return
-    end
-
-    --print("debug dfs", ref.layer.id, t)
-
-    local flag = true --whether have all inputs
-    for _, conn in pairs(ref.o_conns_p) do
-        local p = conn.src.port
-        if (not (ref.err_inputs_b[t][p] or self:out_of_feedrange(t + conn.time))) then
-            flag = false
-            break
-        end
-    end
-    if (flag == false) then
-        return
-    end
-
-    --ok, do back_propagate
-    --print("debug ok, back-propagating(or updating)")
-    if (do_update == false) then
-        self.gconf.timer:tic("tnn_actual_layer_backpropagate")
-        ref.layer:back_propagate(ref.err_inputs_m[t], ref.err_outputs_m[t], ref.inputs_m[t], ref.outputs_m[t], t)
-        self.gconf.timer:toc("tnn_actual_layer_backpropagate")
-        if self.clip_t > 0 then
-            for _, conn in pairs(ref.i_conns_p) do
-                local p = conn.dst.port --port for ref
-                if conn.time ~= 0 then
-                    --print("debug clip_t tnn", ref.id, "port:", p, "clip:", self.clip_t)
-                    ref.err_outputs_m[t][p]:clip(-self.clip_t, self.clip_t)
-                end
-            end
-        end
-    else 
-        --print(ref.err_inputs_m[t][1])
-        self.gconf.timer:tic("tnn_actual_layer_update") 
-        ref.layer:update(ref.err_inputs_m[t], ref.inputs_m[t], ref.outputs_m[t], t)
-        self.gconf.timer:toc("tnn_actual_layer_update") 
-    end
-
-    if (do_update == false and bit.band(self.feeds_now.flagsPack_now[t], bit.bor(nerv.TNN.FC.SEQ_START, nerv.TNN.FC.SEQ_END)) > 0) then --flush cross-border errors
-        for i = 1, self.batch_size do
-            local seq_start = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_START) 
-            local seq_end = bit.band(self.feeds_now.flags_now[t][i], nerv.TNN.FC.SEQ_END) 
-            if (seq_start > 0 or seq_end > 0) then
-                for p, conn in pairs(ref.i_conns_p) do
-                    if ((ref.i_conns_p[p].time > 0 and seq_start > 0) or (ref.i_conns_p[p].time < 0 and seq_end > 0)) then --cross-border, set to zero
-                        ref.err_outputs_m[t][p][i - 1]:fill(0)
-                    end
-                end
-            end
-        end
-    end
-
-    for i = 1, #ref.dim_in do
-        if (ref.err_outputs_b[t][i] == true) then
-            nerv.error("this time's outputs_b should be false")
-        end
-        ref.err_outputs_b[t][i] = true
-    end
-    
-    --try dfs for further layers
-    for _, conn in pairs(ref.i_conns_p) do
-        --print("debug dfs-searching", conn.src.ref.layer.id)
-        conn.src.ref.err_inputs_b[t - conn.time][conn.src.port] = true
-        self:backpropagate_dfs(conn.src.ref, t - conn.time, do_update)
-    end
-end
-
---Return: nerv.ParamRepo
-function TNN:get_params()
-    local param_repos = {}
-    for id, ref in pairs(self.layers) do
-        table.insert(param_repos, ref.layer:get_params())
-    end
-    return nerv.ParamRepo.merge(param_repos)
-end
-