major change: clearer param binding semantics; permit rebinding; enable

resuming from previous training

26 files changed, 526 insertions, 540 deletions

diff --git a/nerv/examples/asr_trainer.lua b/nerv/examples/asr_trainer.lua
index 5001e12..5bf28bd 100644
--- a/nerv/examples/asr_trainer.lua
+++ b/nerv/examples/asr_trainer.lua
@@ -1,19 +1,33 @@
 require 'lfs'
 require 'pl'
 local function build_trainer(ifname)
-    local param_repo = nerv.ParamRepo()
-    param_repo:import(ifname, nil, gconf)
-    local layer_repo = make_layer_repo(param_repo)
-    local network = get_network(layer_repo)
-    local global_transf = get_global_transf(layer_repo)
-    local input_order = get_input_order()
+    local host_param_repo = nerv.ParamRepo()
     local mat_type
+    local src_loc_type
+    local train_loc_type
+    host_param_repo:import(ifname, nil, gconf)
     if gconf.use_cpu then
         mat_type = gconf.mmat_type
+        src_loc_type = nerv.ParamRepo.LOC_TYPES.ON_HOST
+        train_loc_type = nerv.ParamRepo.LOC_TYPES.ON_HOST
     else
         mat_type = gconf.cumat_type
+        src_loc_type = nerv.ParamRepo.LOC_TYPES.ON_HOST
+        train_loc_type = nerv.ParamRepo.LOC_TYPES.ON_DEVICE
     end
-    local iterative_trainer = function (prefix, scp_file, bp)
+    local param_repo = host_param_repo:copy(train_loc_type)
+    local layer_repo = make_layer_repo(param_repo)
+    local network = get_network(layer_repo)
+    local global_transf = get_global_transf(layer_repo)
+    local input_order = get_input_order()
+    local iterative_trainer = function (prefix, scp_file, bp, rebind_param_repo)
+        -- rebind the params if necessary
+        if rebind_param_repo then
+            host_param_repo = rebind_param_repo
+            param_repo = host_param_repo:copy(train_loc_type)
+            layer_repo:rebind(param_repo)
+            rebind_param_repo = nil
+        end
         gconf.randomize = bp
         -- build buffer
         local buffer = make_buffer(make_readers(scp_file, layer_repo))
@@ -66,20 +80,38 @@ local function build_trainer(ifname)
         print_stat(layer_repo)
         mat_type.print_profile()
         mat_type.clear_profile()
-        if (not bp) and prefix ~= nil then
-            nerv.info("writing back...")
-            local fname = string.format("%s_cv%.3f.nerv",
-                            prefix, get_accuracy(layer_repo))
-            network:get_params():export(fname, nil)
+        local fname
+        if (not bp) then
+            host_param_repo = param_repo:copy(src_loc_type)
+            if prefix ~= nil then
+                nerv.info("writing back...")
+                fname = string.format("%s_cv%.3f.nerv",
+                                    prefix, get_accuracy(layer_repo))
+                host_param_repo:export(fname, nil)
+            end
         end
-        return get_accuracy(layer_repo)
+        return get_accuracy(layer_repo), host_param_repo, fname
     end
     return iterative_trainer
 end
 
-local function check_and_add_defaults(spec)
-    for k, v in pairs(spec) do
-        gconf[k] = opts[string.gsub(k, '_', '-')].val or gconf[k] or v
+local function check_and_add_defaults(spec, opts)
+    local function get_opt_val(k)
+        return opts[string.gsub(k, '_', '-')].val
+    end
+    local opt_v = get_opt_val("resume_from")
+    if opt_v then
+        gconf = dofile(opt_v)
+    else
+        for k, v in pairs(spec) do
+            local opt_v = get_opt_val(k)
+            if opt_v ~= nil then
+                gconf[k] = opt_v
+            elseif gconf[k] ~= nil then
+            elseif v ~= nil then
+                gconf[k] = v
+            end
+        end
     end
 end
 
@@ -112,6 +144,13 @@ local function print_gconf()
     end
 end
 
+local function dump_gconf(fname)
+    local f = io.open(fname, "w")
+    f:write("return ")
+    f:write(table.tostring(gconf))
+    f:close()
+end
+
 local trainer_defaults = {
     lrate = 0.8,
     batch_size = 256,
@@ -121,22 +160,26 @@ local trainer_defaults = {
     start_halving_inc = 0.5,
     halving_factor = 0.6,
     end_halving_inc = 0.1,
+    cur_iter = 1,
     min_iter = 1,
     max_iter = 20,
     min_halving = 5,
     do_halving = false,
-    tr_scp = nil,
-    cv_scp = nil,
-    cumat_type = nerv.CuMatrixFloat,
-    mmat_type = nerv.MMatrixFloat,
-    debug = false
+    cumat_tname = "nerv.CuMatrixFloat",
+    mmat_tname = "nerv.MMatrixFloat",
+    debug = false,
 }
 
 local options = make_options(trainer_defaults)
-table.insert(options, {"help", "h", "boolean",
-                        default = false, desc = "show this help information"})
-table.insert(options, {"dir", nil, "string",
-                        default = nil, desc = "specify the working directory"})
+local extra_opt_spec = {
+    {"tr-scp", nil, "string"},
+    {"cv-scp", nil, "string"},
+    {"resume-from", nil, "string"},
+    {"help", "h", "boolean", default = false, desc = "show this help information"},
+    {"dir", nil, "string", desc = "specify the working directory"},
+}
+
+table.extend(options, extra_opt_spec)
 
 arg, opts = nerv.parse_args(arg, options)
 
@@ -155,14 +198,16 @@ Note: config key like aaa_bbbb_cc could be overriden by specifying
 
 ]]--
 
-check_and_add_defaults(trainer_defaults)
+check_and_add_defaults(trainer_defaults, opts)
+gconf.mmat_type = nerv.get_type(gconf.mmat_tname)
+gconf.cumat_type = nerv.get_type(gconf.cumat_tname)
+gconf.use_cpu = econf.use_cpu or false
 
 local pf0 = gconf.initialized_param
-local trainer = build_trainer(pf0)
-local accu_best = trainer(nil, gconf.cv_scp, false)
 local date_pattern = "%Y%m%d%H%M%S"
 local logfile_name = "log"
 local working_dir = opts["dir"].val or string.format("nerv_%s", os.date(date_pattern))
+local rebind_param_repo = nil
 
 print_gconf()
 if not lfs.mkdir(working_dir) then
@@ -173,37 +218,55 @@ dir.copyfile(arg[1], working_dir)
 -- set logfile path
 nerv.set_logfile(path.join(working_dir, logfile_name))
 path.chdir(working_dir)
-nerv.info("initial cross validation: %.3f", accu_best)
-for i = 1, gconf.max_iter do
-    nerv.info("[NN] begin iteration %d with lrate = %.6f", i, gconf.lrate)
-    local accu_tr = trainer(nil, gconf.tr_scp, true)
-    nerv.info("[TR] training set %d: %.3f", i, accu_tr)
-    local accu_new = trainer(
-                        string.format("%s_%s_iter_%d_lr%f_tr%.3f",
-                            string.gsub(
-                                (string.gsub(pf0[1], "(.*/)(.*)", "%2")),
-                                "(.*)%..*", "%1"),
-                            os.date(date_pattern),
-                            i, gconf.lrate,
-                            accu_tr),
-                        gconf.cv_scp, false)
-    nerv.info("[CV] cross validation %d: %.3f", i, accu_new)
-    -- TODO: revert the weights
-    local accu_diff = accu_new - accu_best
-    if gconf.do_halving and
-        accu_diff < gconf.end_halving_inc and
-        i > gconf.min_iter then
-        break
-    end
-    if accu_diff < gconf.start_halving_inc and
-        i >= gconf.min_halving then
-        gconf.do_halving = true
-    end
-    if gconf.do_halving then
-        gconf.lrate = gconf.lrate * gconf.halving_factor
-    end
-    if accu_new > accu_best then
-        accu_best = accu_new
-    end
+
+-- start the training
+local trainer = build_trainer(pf0)
+local pr_prev
+gconf.accu_best, pr_prev = trainer(nil, gconf.cv_scp, false)
+nerv.info("initial cross validation: %.3f", gconf.accu_best)
+for i = gconf.cur_iter, gconf.max_iter do
+    local stop = false
+    gconf.cur_iter = i
+    dump_gconf(string.format("iter_%d.meta", i))
+    repeat -- trick to implement `continue` statement
+        nerv.info("[NN] begin iteration %d with lrate = %.6f", i, gconf.lrate)
+        local accu_tr = trainer(nil, gconf.tr_scp, true, rebind_param_repo)
+        nerv.info("[TR] training set %d: %.3f", i, accu_tr)
+        local param_prefix = string.format("%s_%s_iter_%d_lr%f_tr%.3f",
+                                string.gsub(
+                                    (string.gsub(pf0[1], "(.*/)(.*)", "%2")),
+                                    "(.*)%..*", "%1"),
+                                os.date(date_pattern),
+                                i, gconf.lrate,
+                                accu_tr)
+        local accu_new, pr_new, param_fname = trainer(param_prefix, gconf.cv_scp, false)
+        nerv.info("[CV] cross validation %d: %.3f", i, accu_new)
+        local accu_prev = gconf.accu_best
+        if accu_new < gconf.accu_best then
+            nerv.info("rejecting the trained params, rollback to the previous one")
+            file.move(param_fname, param_fname .. ".rejected")
+            rebind_param_repo = pr_prev
+            break -- `continue` equivalent
+        else
+            nerv.info("accepting the trained params")
+            gconf.accu_best = accu_new
+            pr_prev = pr_new
+            gconf.initialized_param = {path.join(path.currentdir(), param_fname)}
+        end
+        if gconf.do_halving and
+            gconf.accu_best - accu_prev < gconf.end_halving_inc and
+            i > gconf.min_iter then
+            stop = true
+            break
+        end
+        if gconf.accu_best - accu_prev < gconf.start_halving_inc and
+            i >= gconf.min_halving then
+            gconf.do_halving = true
+        end
+        if gconf.do_halving then
+            gconf.lrate = gconf.lrate * gconf.halving_factor
+        end
+    until true
+    if stop then break end
 --    nerv.Matrix.print_profile()
 end
diff --git a/nerv/examples/swb_baseline.lua b/nerv/examples/swb_baseline.lua
index 4cb2389..0ce8468 100644
--- a/nerv/examples/swb_baseline.lua
+++ b/nerv/examples/swb_baseline.lua
@@ -7,8 +7,7 @@ gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9,
         cv_scp = "/slfs1/users/mfy43/swb_ivec/train_cv.scp",
         htk_conf = "/slfs1/users/mfy43/swb_ivec/plp_0_d_a.conf",
         initialized_param = {"/slfs1/users/mfy43/swb_init.nerv",
-                            "/slfs1/users/mfy43/swb_global_transf.nerv"},
-        debug = false}
+                            "/slfs1/users/mfy43/swb_global_transf.nerv"}}
 
 function make_layer_repo(param_repo)
     local layer_repo = nerv.LayerRepo(
@@ -16,51 +15,51 @@ function make_layer_repo(param_repo)
         -- global transf
         ["nerv.BiasLayer"] =
         {
-            blayer1 = {{bias = "bias1"}, {dim_in = {429}, dim_out = {429}}},
-            blayer2 = {{bias = "bias2"}, {dim_in = {429}, dim_out = {429}}}
+            blayer1 = {dim_in = {429}, dim_out = {429}, params = {bias = "bias1"}},
+            blayer2 = {dim_in = {429}, dim_out = {429}, params = {bias = "bias2"}}
         },
         ["nerv.WindowLayer"] =
         {
-            wlayer1 = {{window = "window1"}, {dim_in = {429}, dim_out = {429}}},
-            wlayer2 = {{window = "window2"}, {dim_in = {429}, dim_out = {429}}}
+            wlayer1 = {dim_in = {429}, dim_out = {429}, params = {window = "window1"}},
+            wlayer2 = {dim_in = {429}, dim_out = {429}, params = {window = "window2"}}
         },
         -- biased linearity
         ["nerv.AffineLayer"] =
         {
-            affine0 = {{ltp = "affine0_ltp", bp = "affine0_bp"},
-            {dim_in = {429}, dim_out = {2048}}},
-            affine1 = {{ltp = "affine1_ltp", bp = "affine1_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine2 = {{ltp = "affine2_ltp", bp = "affine2_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine3 = {{ltp = "affine3_ltp", bp = "affine3_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine4 = {{ltp = "affine4_ltp", bp = "affine4_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine5 = {{ltp = "affine5_ltp", bp = "affine5_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine6 = {{ltp = "affine6_ltp", bp = "affine6_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine7 = {{ltp = "affine7_ltp", bp = "affine7_bp"},
-            {dim_in = {2048}, dim_out = {3001}}}
+            affine0 = {dim_in = {429}, dim_out = {2048},
+                        params = {ltp = "affine0_ltp", bp = "affine0_bp"}},
+            affine1 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine1_ltp", bp = "affine1_bp"}},
+            affine2 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine2_ltp", bp = "affine2_bp"}},
+            affine3 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine3_ltp", bp = "affine3_bp"}},
+            affine4 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine4_ltp", bp = "affine4_bp"}},
+            affine5 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine5_ltp", bp = "affine5_bp"}},
+            affine6 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine6_ltp", bp = "affine6_bp"}},
+            affine7 = {dim_in = {2048}, dim_out = {3001},
+                        params = {ltp = "affine7_ltp", bp = "affine7_bp"}}
         },
         ["nerv.SigmoidLayer"] =
         {
-            sigmoid0 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid1 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid2 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid3 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid4 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid5 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid6 = {{}, {dim_in = {2048}, dim_out = {2048}}}
+            sigmoid0 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid1 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid2 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid3 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid4 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid5 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid6 = {dim_in = {2048}, dim_out = {2048}}
         },
         ["nerv.SoftmaxCELayer"] = -- softmax + ce criterion layer for finetune output
         {
-            ce_crit = {{}, {dim_in = {3001, 1}, dim_out = {1}, compressed = true}}
+            ce_crit = {dim_in = {3001, 1}, dim_out = {1}, compressed = true}
         },
         ["nerv.SoftmaxLayer"] = -- softmax for decode output
         {
-            softmax = {{}, {dim_in = {3001}, dim_out = {3001}}}
+            softmax = {dim_in = {3001}, dim_out = {3001}}
         }
     }, param_repo, gconf)
 
@@ -68,7 +67,7 @@ function make_layer_repo(param_repo)
     {
         ["nerv.DAGLayer"] =
         {
-            global_transf = {{}, {
+            global_transf = {
                 dim_in = {429}, dim_out = {429},
                 sub_layers = layer_repo,
                 connections = {
@@ -78,8 +77,8 @@ function make_layer_repo(param_repo)
                     ["blayer2[1]"] = "wlayer2[1]",
                     ["wlayer2[1]"] = "<output>[1]"
                 }
-            }},
-            main = {{}, {
+            },
+            main = {
                 dim_in = {429}, dim_out = {3001},
                 sub_layers = layer_repo,
                 connections = {
@@ -100,7 +99,7 @@ function make_layer_repo(param_repo)
                     ["sigmoid6[1]"] = "affine7[1]",
                     ["affine7[1]"] = "<output>[1]"
                 }
-            }}
+            }
         }
     }, param_repo, gconf)
 
@@ -108,7 +107,7 @@ function make_layer_repo(param_repo)
     {
         ["nerv.DAGLayer"] =
         {
-            ce_output = {{}, {
+            ce_output = {
                 dim_in = {429, 1}, dim_out = {1},
                 sub_layers = layer_repo,
                 connections = {
@@ -117,8 +116,8 @@ function make_layer_repo(param_repo)
                     ["<input>[2]"] = "ce_crit[2]",
                     ["ce_crit[1]"] = "<output>[1]"
                 }
-            }},
-            softmax_output = {{}, {
+            },
+            softmax_output = {
                 dim_in = {429}, dim_out = {3001},
                 sub_layers = layer_repo,
                 connections = {
@@ -126,7 +125,7 @@ function make_layer_repo(param_repo)
                     ["main[1]"] = "softmax[1]",
                     ["softmax[1]"] = "<output>[1]"
                 }
-            }}
+            }
         }
     }, param_repo, gconf)
 
diff --git a/nerv/examples/swb_baseline2.lua b/nerv/examples/swb_baseline2.lua
index b0b9689..8b5ebb1 100644
--- a/nerv/examples/swb_baseline2.lua
+++ b/nerv/examples/swb_baseline2.lua
@@ -7,8 +7,7 @@ gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9,
         cv_scp = "/speechlab/users/mfy43/swb50/train_cv.scp",
         htk_conf = "/speechlab/users/mfy43/swb50/plp_0_d_a.conf",
         initialized_param = {"/speechlab/users/mfy43/swb50/swb_init.nerv",
-                            "/speechlab/users/mfy43/swb50/swb_global_transf.nerv"},
-        debug = false}
+                            "/speechlab/users/mfy43/swb50/swb_global_transf.nerv"}}
 
 function make_layer_repo(param_repo)
     local layer_repo = nerv.LayerRepo(
@@ -16,51 +15,51 @@ function make_layer_repo(param_repo)
         -- global transf
         ["nerv.BiasLayer"] =
         {
-            blayer1 = {{bias = "bias1"}, {dim_in = {429}, dim_out = {429}}},
-            blayer2 = {{bias = "bias2"}, {dim_in = {429}, dim_out = {429}}}
+            blayer1 = {dim_in = {429}, dim_out = {429}, params = {bias = "bias1"}},
+            blayer2 = {dim_in = {429}, dim_out = {429}, params = {bias = "bias2"}}
         },
         ["nerv.WindowLayer"] =
         {
-            wlayer1 = {{window = "window1"}, {dim_in = {429}, dim_out = {429}}},
-            wlayer2 = {{window = "window2"}, {dim_in = {429}, dim_out = {429}}}
+            wlayer1 = {dim_in = {429}, dim_out = {429}, params = {window = "window1"}},
+            wlayer2 = {dim_in = {429}, dim_out = {429}, params = {window = "window2"}}
         },
         -- biased linearity
         ["nerv.AffineLayer"] =
         {
-            affine0 = {{ltp = "affine0_ltp", bp = "affine0_bp"},
-            {dim_in = {429}, dim_out = {2048}}},
-            affine1 = {{ltp = "affine1_ltp", bp = "affine1_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine2 = {{ltp = "affine2_ltp", bp = "affine2_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine3 = {{ltp = "affine3_ltp", bp = "affine3_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine4 = {{ltp = "affine4_ltp", bp = "affine4_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine5 = {{ltp = "affine5_ltp", bp = "affine5_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine6 = {{ltp = "affine6_ltp", bp = "affine6_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine7 = {{ltp = "affine7_ltp", bp = "affine7_bp"},
-            {dim_in = {2048}, dim_out = {3001}}}
+            affine0 = {dim_in = {429}, dim_out = {2048},
+                        params = {ltp = "affine0_ltp", bp = "affine0_bp"}},
+            affine1 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine1_ltp", bp = "affine1_bp"}},
+            affine2 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine2_ltp", bp = "affine2_bp"}},
+            affine3 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine3_ltp", bp = "affine3_bp"}},
+            affine4 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine4_ltp", bp = "affine4_bp"}},
+            affine5 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine5_ltp", bp = "affine5_bp"}},
+            affine6 = {dim_in = {2048}, dim_out = {2048},
+                        params = {ltp = "affine6_ltp", bp = "affine6_bp"}},
+            affine7 = {dim_in = {2048}, dim_out = {3001},
+                        params = {ltp = "affine7_ltp", bp = "affine7_bp"}}
         },
         ["nerv.SigmoidLayer"] =
         {
-            sigmoid0 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid1 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid2 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid3 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid4 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid5 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid6 = {{}, {dim_in = {2048}, dim_out = {2048}}}
+            sigmoid0 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid1 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid2 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid3 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid4 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid5 = {dim_in = {2048}, dim_out = {2048}},
+            sigmoid6 = {dim_in = {2048}, dim_out = {2048}}
         },
         ["nerv.SoftmaxCELayer"] = -- softmax + ce criterion layer for finetune output
         {
-            ce_crit = {{}, {dim_in = {3001, 1}, dim_out = {1}, compressed = true}}
+            ce_crit = {dim_in = {3001, 1}, dim_out = {1}, compressed = true}
         },
         ["nerv.SoftmaxLayer"] = -- softmax for decode output
         {
-            softmax = {{}, {dim_in = {3001}, dim_out = {3001}}}
+            softmax = {dim_in = {3001}, dim_out = {3001}}
         }
     }, param_repo, gconf)
 
@@ -68,7 +67,7 @@ function make_layer_repo(param_repo)
     {
         ["nerv.DAGLayer"] =
         {
-            global_transf = {{}, {
+            global_transf = {
                 dim_in = {429}, dim_out = {429},
                 sub_layers = layer_repo,
                 connections = {
@@ -78,8 +77,8 @@ function make_layer_repo(param_repo)
                     ["blayer2[1]"] = "wlayer2[1]",
                     ["wlayer2[1]"] = "<output>[1]"
                 }
-            }},
-            main = {{}, {
+            },
+            main = {
                 dim_in = {429}, dim_out = {3001},
                 sub_layers = layer_repo,
                 connections = {
@@ -100,7 +99,7 @@ function make_layer_repo(param_repo)
                     ["sigmoid6[1]"] = "affine7[1]",
                     ["affine7[1]"] = "<output>[1]"
                 }
-            }}
+            }
         }
     }, param_repo, gconf)
 
@@ -108,7 +107,7 @@ function make_layer_repo(param_repo)
     {
         ["nerv.DAGLayer"] =
         {
-            ce_output = {{}, {
+            ce_output = {
                 dim_in = {429, 1}, dim_out = {1},
                 sub_layers = layer_repo,
                 connections = {
@@ -117,8 +116,8 @@ function make_layer_repo(param_repo)
                     ["<input>[2]"] = "ce_crit[2]",
                     ["ce_crit[1]"] = "<output>[1]"
                 }
-            }},
-            softmax_output = {{}, {
+            },
+            softmax_output = {
                 dim_in = {429}, dim_out = {3001},
                 sub_layers = layer_repo,
                 connections = {
@@ -126,7 +125,7 @@ function make_layer_repo(param_repo)
                     ["main[1]"] = "softmax[1]",
                     ["softmax[1]"] = "<output>[1]"
                 }
-            }}
+            }
         }
     }, param_repo, gconf)
 
diff --git a/nerv/examples/swb_baseline_basic.lua b/nerv/examples/swb_baseline_basic.lua
deleted file mode 100644
index 71f04a3..0000000
--- a/nerv/examples/swb_baseline_basic.lua
+++ /dev/null
@@ -1,162 +0,0 @@
-require 'htk_io'
-gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9,
-        cumat_type = nerv.CuMatrixFloat,
-        mmat_type = nerv.MMatrixFloat,
-        frm_ext = 5,
-        frm_trim = 5,
-        tr_scp = "/slfs1/users/mfy43/swb_ivec/train_bp.scp",
-        cv_scp = "/slfs1/users/mfy43/swb_ivec/train_cv.scp",
-        htk_conf = "/slfs1/users/mfy43/swb_ivec/plp_0_d_a.conf",
-        initialized_param = {"/slfs1/users/mfy43/swb_init.nerv",
-                "/slfs1/users/mfy43/swb_global_transf.nerv"},
-        debug = false}
-
-function make_layer_repo(param_repo)
-    local layer_repo = nerv.LayerRepo(
-    {
-        -- global transf
-        ["nerv.BiasLayer"] =
-        {
-            blayer1 = {{bias = "bias1"}, {dim_in = {429}, dim_out = {429}}},
-            blayer2 = {{bias = "bias2"}, {dim_in = {429}, dim_out = {429}}}
-        },
-        ["nerv.WindowLayer"] =
-        {
-            wlayer1 = {{window = "window1"}, {dim_in = {429}, dim_out = {429}}},
-            wlayer2 = {{window = "window2"}, {dim_in = {429}, dim_out = {429}}}
-        },
-        -- biased linearity
-        ["nerv.AffineLayer"] =
-        {
-            affine0 = {{ltp = "affine0_ltp", bp = "affine0_bp"},
-            {dim_in = {429}, dim_out = {2048}}},
-            affine1 = {{ltp = "affine1_ltp", bp = "affine1_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine2 = {{ltp = "affine2_ltp", bp = "affine2_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine3 = {{ltp = "affine3_ltp", bp = "affine3_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine4 = {{ltp = "affine4_ltp", bp = "affine4_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine5 = {{ltp = "affine5_ltp", bp = "affine5_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine6 = {{ltp = "affine6_ltp", bp = "affine6_bp"},
-            {dim_in = {2048}, dim_out = {2048}}},
-            affine7 = {{ltp = "affine7_ltp", bp = "affine7_bp"},
-            {dim_in = {2048}, dim_out = {3001}}}
-        },
-        ["nerv.SigmoidLayer"] =
-        {
-            sigmoid0 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid1 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid2 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid3 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid4 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid5 = {{}, {dim_in = {2048}, dim_out = {2048}}},
-            sigmoid6 = {{}, {dim_in = {2048}, dim_out = {2048}}}
-        },
-        ["nerv.SoftmaxCELayer"] =
-        {
-            ce_crit = {{}, {dim_in = {3001, 1}, dim_out = {1}, compressed = true}}
-        }
-    }, param_repo, gconf)
-
-    layer_repo:add_layers(
-    {
-        ["nerv.DAGLayer"] =
-        {
-            global_transf = {{}, {
-                dim_in = {429}, dim_out = {429},
-                sub_layers = layer_repo,
-                connections = {
-                    ["<input>[1]"] = "blayer1[1]",
-                    ["blayer1[1]"] = "wlayer1[1]",
-                    ["wlayer1[1]"] = "blayer2[1]",
-                    ["blayer2[1]"] = "wlayer2[1]",
-                    ["wlayer2[1]"] = "<output>[1]"
-                }
-            }},
-            main = {{}, {
-                dim_in = {429, 1}, dim_out = {1},
-                sub_layers = layer_repo,
-                connections = {
-                    ["<input>[1]"] = "affine0[1]",
-                    ["affine0[1]"] = "sigmoid0[1]",
-                    ["sigmoid0[1]"] = "affine1[1]",
-                    ["affine1[1]"] = "sigmoid1[1]",
-                    ["sigmoid1[1]"] = "affine2[1]",
-                    ["affine2[1]"] = "sigmoid2[1]",
-                    ["sigmoid2[1]"] = "affine3[1]",
-                    ["affine3[1]"] = "sigmoid3[1]",
-                    ["sigmoid3[1]"] = "affine4[1]",
-                    ["affine4[1]"] = "sigmoid4[1]",
-                    ["sigmoid4[1]"] = "affine5[1]",
-                    ["affine5[1]"] = "sigmoid5[1]",
-                    ["sigmoid5[1]"] = "affine6[1]",
-                    ["affine6[1]"] = "sigmoid6[1]",
-                    ["sigmoid6[1]"] = "affine7[1]",
-                    ["affine7[1]"] = "ce_crit[1]",
-                    ["<input>[2]"] = "ce_crit[2]",
-                    ["ce_crit[1]"] = "<output>[1]"
-                }
-            }}
-        }
-    }, param_repo, gconf)
-    return layer_repo
-end
-
-function get_network(layer_repo)
-    return layer_repo:get_layer("main")
-end
-
-function make_readers(scp_file, layer_repo)
-    return {
-                {reader = nerv.TNetReader(gconf,
-                    {
-                        id = "main_scp",
-                        scp_file = scp_file,
-                        conf_file = gconf.htk_conf,
-                        frm_ext = gconf.frm_ext,
-                        mlfs = {
-                            phone_state = {
-                                file = "/slfs1/users/mfy43/swb_ivec/ref.mlf",
-                                format = "map",
-                                format_arg = "/slfs1/users/mfy43/swb_ivec/dict",
-                                dir = "*/",
-                                ext = "lab"
-                            }
-                        }
-                    }),
-                data = {main_scp = 429, phone_state = 1}}
-            }
-end
-
-function make_buffer(readers)
-    return nerv.SGDBuffer(gconf,
-        {
-            buffer_size = gconf.buffer_size,
-            randomize = gconf.randomize,
-            readers = readers
-        })
-end
-
-function get_input_order()
-    return {{id = "main_scp", global_transf = true},
-            {id = "phone_state"}}
-end
-
-function get_accuracy(layer_repo)
-    local ce_crit = layer_repo:get_layer("ce_crit")
-    return ce_crit.total_correct / ce_crit.total_frames * 100
-end
-
-function print_stat(layer_repo)
-    local ce_crit = layer_repo:get_layer("ce_crit")
-    nerv.info("*** training stat begin ***")
-    nerv.printf("cross entropy:\t\t%.8f\n", ce_crit.total_ce)
-    nerv.printf("correct:\t\t%d\n", ce_crit.total_correct)
-    nerv.printf("frames:\t\t\t%d\n", ce_crit.total_frames)
-    nerv.printf("err/frm:\t\t%.8f\n", ce_crit.total_ce / ce_crit.total_frames)
-    nerv.printf("accuracy:\t\t%.3f%%\n", get_accuracy(layer_repo))
-    nerv.info("*** training stat end ***")
-end
diff --git a/nerv/examples/timit_baseline2.lua b/nerv/examples/timit_baseline2.lua
index 103d89d..2d144b5 100644
--- a/