From ad704f2623cc9e0a5d702434bfdebc345465ca12 Mon Sep 17 00:00:00 2001 From: Determinant Date: Wed, 2 Mar 2016 18:24:09 +0800 Subject: major changes in asr_trainer.lua; unified settings in `gconf` --- nerv/examples/asr_trainer.lua | 104 +++++++++++++++++++++++++++++++------- nerv/examples/swb_baseline.lua | 7 ++- nerv/examples/swb_baseline2.lua | 7 ++- nerv/examples/timit_baseline2.lua | 9 ++-- nerv/init.lua | 22 ++++---- nerv/io/sgd_buffer.lua | 7 +-- nerv/nerv | 11 ++-- nerv/test/parse_args.lua | 12 ++--- 8 files changed, 126 insertions(+), 53 deletions(-) diff --git a/nerv/examples/asr_trainer.lua b/nerv/examples/asr_trainer.lua index 3fa2653..684ea30 100644 --- a/nerv/examples/asr_trainer.lua +++ b/nerv/examples/asr_trainer.lua @@ -1,4 +1,4 @@ -function build_trainer(ifname) +local function build_trainer(ifname) local param_repo = nerv.ParamRepo() param_repo:import(ifname, nil, gconf) local layer_repo = make_layer_repo(param_repo) @@ -75,24 +75,91 @@ function build_trainer(ifname) return iterative_trainer end +local function check_and_add_defaults(spec) + for k, v in pairs(spec) do + gconf[k] = opts[string.gsub(k, '_', '-')].val or gconf[k] or v + end +end + +local function make_options(spec) + local options = {} + for k, v in pairs(spec) do + table.insert(options, + {string.gsub(k, '_', '-'), nil, type(v), default = v}) + end + return options +end + +local function print_help(options) + nerv.printf("Usage: [options] network_config.lua\n") + nerv.print_usage(options) +end + +local function print_gconf() + local key_maxlen = 0 + for k, v in pairs(gconf) do + key_maxlen = math.max(key_maxlen, #k or 0) + end + local function pattern_gen() + return string.format("%%-%ds = %%s\n", key_maxlen) + end + nerv.info("ready to train with the following gconf settings:") + nerv.printf(pattern_gen(), "Key", "Value") + for k, v in pairs(gconf) do + nerv.printf(pattern_gen(), k or "", v or "") + end +end + +local trainer_defaults = { + lrate = 0.8, + batch_size = 256, + buffer_size = 81920, + wcost = 1e-6, + momentum = 0.9, + start_halving_inc = 0.5, + halving_factor = 0.6, + end_halving_inc = 0.1, + min_iter = 1, + max_iter = 20, + min_halving = 5, + do_halving = false, + tr_scp = nil, + cv_scp = nil, + cumat_type = nerv.CuMatrixFloat, + mmat_type = nerv.MMatrixFloat, + debug = false +} + +local options = make_options(trainer_defaults) +table.insert(options, {"help", "h", "boolean", + default = false, desc = "show this help information"}) + +arg, opts = nerv.parse_args(arg, options) + +if #arg < 1 or opts["help"].val then + print_help(options) + return +end + dofile(arg[1]) -start_halving_inc = 0.5 -halving_factor = 0.6 -end_halving_inc = 0.1 -min_iter = 1 -max_iter = 20 -min_halving = 5 -gconf.batch_size = 256 -gconf.buffer_size = 81920 + +--[[ + +Rule: command-line option overrides network config overrides trainer default. +Note: config key like aaa_bbbb_cc could be overriden by specifying +--aaa-bbbb-cc to command-line arguments. + +]]-- + +check_and_add_defaults(trainer_defaults) local pf0 = gconf.initialized_param local trainer = build_trainer(pf0) ---local trainer = build_trainer("c3.nerv") local accu_best = trainer(nil, gconf.cv_scp, false) -local do_halving = false +print_gconf() nerv.info("initial cross validation: %.3f", accu_best) -for i = 1, max_iter do +for i = 1, gconf.max_iter do nerv.info("[NN] begin iteration %d with lrate = %.6f", i, gconf.lrate) local accu_tr = trainer(nil, gconf.tr_scp, true) nerv.info("[TR] training set %d: %.3f", i, accu_tr) @@ -108,14 +175,17 @@ for i = 1, max_iter do nerv.info("[CV] cross validation %d: %.3f", i, accu_new) -- TODO: revert the weights local accu_diff = accu_new - accu_best - if do_halving and accu_diff < end_halving_inc and i > min_iter then + if gconf.do_halving and + accu_diff < gconf.end_halving_inc and + i > gconf.min_iter then break end - if accu_diff < start_halving_inc and i >= min_halving then - do_halving = true + if accu_diff < gconf.start_halving_inc and + i >= gconf.min_halving then + gconf.do_halving = true end - if do_halving then - gconf.lrate = gconf.lrate * halving_factor + if gconf.do_halving then + gconf.lrate = gconf.lrate * gconf.halving_factor end if accu_new > accu_best then accu_best = accu_new diff --git a/nerv/examples/swb_baseline.lua b/nerv/examples/swb_baseline.lua index cacc401..4cb2389 100644 --- a/nerv/examples/swb_baseline.lua +++ b/nerv/examples/swb_baseline.lua @@ -1,7 +1,5 @@ require 'htk_io' gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, rearrange = true, -- just to make the context order consistent with old results, deprecated frm_ext = 5, frm_trim = 5, -- trim the first and last 5 frames, TNet just does this, deprecated @@ -173,6 +171,7 @@ function make_buffer(readers) return nerv.SGDBuffer(gconf, { buffer_size = gconf.buffer_size, + batch_size = gconf.batch_size, randomize = gconf.randomize, readers = readers, use_gpu = true @@ -184,6 +183,10 @@ function get_input_order() {id = "phone_state"}} end +function get_decode_input_order() + return {{id = "main_scp", global_transf = true}} +end + function get_accuracy(layer_repo) local ce_crit = layer_repo:get_layer("ce_crit") return ce_crit.total_correct / ce_crit.total_frames * 100 diff --git a/nerv/examples/swb_baseline2.lua b/nerv/examples/swb_baseline2.lua index 0e2a6e0..b0b9689 100644 --- a/nerv/examples/swb_baseline2.lua +++ b/nerv/examples/swb_baseline2.lua @@ -1,7 +1,5 @@ require 'htk_io' gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, rearrange = true, -- just to make the context order consistent with old results, deprecated frm_ext = 5, frm_trim = 5, -- trim the first and last 5 frames, TNet just does this, deprecated @@ -173,6 +171,7 @@ function make_buffer(readers) return nerv.SGDBuffer(gconf, { buffer_size = gconf.buffer_size, + batch_size = gconf.batch_size, randomize = gconf.randomize, readers = readers, use_gpu = true @@ -184,6 +183,10 @@ function get_input_order() {id = "phone_state"}} end +function get_decode_input_order() + return {{id = "main_scp", global_transf = true}} +end + function get_accuracy(layer_repo) local ce_crit = layer_repo:get_layer("ce_crit") return ce_crit.total_correct / ce_crit.total_frames * 100 diff --git a/nerv/examples/timit_baseline2.lua b/nerv/examples/timit_baseline2.lua index 174b9e7..103d89d 100644 --- a/nerv/examples/timit_baseline2.lua +++ b/nerv/examples/timit_baseline2.lua @@ -1,8 +1,5 @@ require 'kaldi_io' -gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9, - cumat_type = nerv.CuMatrixFloat, - mmat_type = nerv.MMatrixFloat, - frm_ext = 5, +gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9, frm_ext = 5, tr_scp = "ark:/speechlab/tools/KALDI/kaldi-master/src/featbin/copy-feats " .. "scp:/speechlab/users/mfy43/timit/s5/exp/dnn4_nerv_prepare/train.scp ark:- |", cv_scp = "ark:/speechlab/tools/KALDI/kaldi-master/src/featbin/copy-feats " .. @@ -11,8 +8,7 @@ gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9, "/speechlab/users/mfy43/timit/s5/exp/dnn4_nerv_prepare/nnet_output.nerv", "/speechlab/users/mfy43/timit/s5/exp/dnn4_nerv_prepare/nnet_trans.nerv"}, decode_param = {"/speechlab/users/mfy43/timit/nnet_init_20160229015745_iter_13_lr0.013437_tr72.434_cv58.729.nerv", - "/speechlab/users/mfy43/timit/s5/exp/dnn4_nerv_prepare/nnet_trans.nerv"}, - debug = false} + "/speechlab/users/mfy43/timit/s5/exp/dnn4_nerv_prepare/nnet_trans.nerv"}} function make_layer_repo(param_repo) local layer_repo = nerv.LayerRepo( @@ -183,6 +179,7 @@ function make_buffer(readers) return nerv.SGDBuffer(gconf, { buffer_size = gconf.buffer_size, + batch_size = gconf.batch_size, randomize = gconf.randomize, readers = readers, use_gpu = true diff --git a/nerv/init.lua b/nerv/init.lua index d72d8b8..a5b032c 100644 --- a/nerv/init.lua +++ b/nerv/init.lua @@ -182,15 +182,15 @@ end -- value and description of the option. -- -- An example of specification: --- {{"aaa", "a", "bool", default = false, desc = "an option called aaa"}, --- {"bbb", "b", "bool", default = true, desc = "bbb is set to be true if --bbb=no does not present"}, +-- {{"aaa", "a", "boolean", default = false, desc = "an option called aaa"}, +-- {"bbb", "b", "boolean", default = true, desc = "bbb is set to be true if --bbb=no does not present"}, -- {"ccc", nil, "int", default = 0, desc = "ccc expects an integeral value"}}` -- -- @return args, opts The non-option arguments and parsed options. `opts` is -- again a list of tables, each of which corresponds to one table in parameter -- `options`. The parsed value could be accessed by `opts["aaa"].val` (which is -- `true` if "--aaa" or "-a" is specified). -function nerv.parse_args(argv, options) +function nerv.parse_args(argv, options, unordered) local is_opt_exp = "^[-](.*)$" local sim_opt_exp = "^[-]([a-z]+)$" local opt_exp = "^[-][-]([^=]+)$" @@ -198,6 +198,7 @@ function nerv.parse_args(argv, options) local opts = {} local sopts = {} local args = {} + local arg_start = false local function err() nerv.error("invalid format of option specification") end @@ -215,7 +216,7 @@ function nerv.parse_args(argv, options) val = v.default} if opt_short ~= nil then if type(opt_short) ~= "string" or #opt_short ~= 1 then err() end - if opt_type ~= "bool" then + if opt_type ~= "boolean" then nerv.error("only boolean option could have short form") end sopts[opt_short] = opt_meta @@ -226,7 +227,7 @@ function nerv.parse_args(argv, options) end end for _, token in ipairs(argv) do - if token:match(is_opt_exp) then + if ((not arg_start) or unordered) and token:match(is_opt_exp) then local k = token:match(sim_opt_exp) if k then for c in k:gmatch"." do @@ -242,7 +243,7 @@ function nerv.parse_args(argv, options) if opts[k] == nil then nerv.error("invalid option %s", token) end - if opts[k].type ~= "bool" then + if opts[k].type ~= "boolean" then nerv.error("invalid option --%s: " .. "a %s value needs to be specified", k, opts[k].type) @@ -255,13 +256,13 @@ function nerv.parse_args(argv, options) if opts[k] == nil then nerv.error("invalid option %s", token) end - if opts[k].type == "bool" then + if opts[k].type == "boolean" then if v == "yes" then opts[k].val = true elseif v == "no" then opts[k].val = false else - nerv.error("bool value should be \"yes\" or \"no\"") + nerv.error("boolean value should be \"yes\" or \"no\"") end elseif opts[k].type == "int" then local t = tonumber(v) @@ -269,11 +270,11 @@ function nerv.parse_args(argv, options) if t == nil or math.floor(t) ~= t then nerv.error("int value is expected") end - elseif opts[k].type == "float" then + elseif opts[k].type == "number" then local t = tonumber(v) opts[k].val = t if t == nil then - nerv.error("float value is expected") + nerv.error("numeric value is expected") end elseif opts[k].type == "string" then opts[k].val = v @@ -287,6 +288,7 @@ function nerv.parse_args(argv, options) end else table.insert(args, token) + arg_start = true end end return args, opts diff --git a/nerv/io/sgd_buffer.lua b/nerv/io/sgd_buffer.lua index 3cf4f5a..d78f6d1 100644 --- a/nerv/io/sgd_buffer.lua +++ b/nerv/io/sgd_buffer.lua @@ -2,8 +2,9 @@ local SGDBuffer = nerv.class("nerv.SGDBuffer", "nerv.DataBuffer") function SGDBuffer:__init(global_conf, buffer_conf) self.gconf = global_conf + self.batch_size = buffer_conf.batch_size self.buffer_size = math.floor(buffer_conf.buffer_size / - global_conf.batch_size) * global_conf.batch_size + self.batch_size) * self.batch_size self.randomize = buffer_conf.randomize self.consume = buffer_conf.consume local cumat_type = global_conf.cumat_type @@ -112,11 +113,11 @@ function SGDBuffer:saturate() end self.rand_map = self.perm_gen(self.tail) -- generate shuffled index collectgarbage("collect") - return self.tail >= self.gconf.batch_size + return self.tail >= self.batch_size end function SGDBuffer:get_data() - local batch_size = self.gconf.batch_size + local batch_size = self.batch_size if self.head >= self.tail then -- buffer is empty local t = os.clock() if (not self:saturate()) and (not self.consume) then diff --git a/nerv/nerv b/nerv/nerv index 0b75a9b..f73d517 100644 --- a/nerv/nerv +++ b/nerv/nerv @@ -1,8 +1,8 @@ #! /usr/bin/env luajit require 'nerv' -local options = {{"help", "h", "bool", default = false, desc = "print this help message"}, - {"use-cpu", "c", "bool", default = false, desc = "use CPU by default (instead of gpu by default)"}, - {"select-gpu", nil, "int", default = nil, desc = "select the GPU for computation, fallback to auto mode if not specified"}} +local options = {{"help", "h", "boolean", default = false, desc = "print this help message"}, + {"use-cpu", "c", "boolean", default = false, desc = "use CPU by default (instead of gpu by default)"}, + {"select-gpu", nil, "int", default = -1, desc = "select the GPU for computation, fallback to auto mode if not specified"}} local function print_help() nerv.printf("Usage: [options] script.lua\n") @@ -24,10 +24,7 @@ local function _add_profile_method(cls) end if not opts["use-cpu"].val then - local dev = -1 - if opts["select-gpu"].val then - dev = opts["select-gpu"].val - end + local dev = opts["select-gpu"].val nerv.info("automatically initialize a default CuContext...") nerv.CuMatrix._default_context = nerv.CuContext(dev) nerv.info("the default CuContext is ok") diff --git a/nerv/test/parse_args.lua b/nerv/test/parse_args.lua index 0d280a1..34ad55e 100644 --- a/nerv/test/parse_args.lua +++ b/nerv/test/parse_args.lua @@ -1,9 +1,9 @@ -local options = {{"abandon", "a", "bool", default = false, desc = "abandon your belief"}, - {"bullshit", "b", "bool", default = false, desc = "start to bullshit"}, - {"cheat", "c", "bool", default = false, desc = "try to cheat"}, - {"delete", "d", "bool", default = false, desc = "remove everything"}, - {"hehe", "h", "bool", default = false, desc = "233333"}, - {"oh", "o", "bool", default = true, desc = "oh yes!"}, +local options = {{"abandon", "a", "boolean", default = false, desc = "abandon your belief"}, + {"bullshit", "b", "boolean", default = false, desc = "start to bullshit"}, + {"cheat", "c", "boolean", default = false, desc = "try to cheat"}, + {"delete", "d", "boolean", default = false, desc = "remove everything"}, + {"hehe", "h", "boolean", default = false, desc = "233333"}, + {"oh", "o", "boolean", default = true, desc = "oh yes!"}, {"uid", nil, "int", desc = "user uid"}, {"str", nil, "string", desc = "test string"}} -- cgit v1.2.3-70-g09d2