From 5bcd5d79875587b08d598cc08bd5f8b1f5e14a23 Mon Sep 17 00:00:00 2001 From: Determinant Date: Sat, 6 Jun 2015 15:48:04 +0800 Subject: ... --- examples/asr_trainer.lua | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'examples') diff --git a/examples/asr_trainer.lua b/examples/asr_trainer.lua index b43a547..d72b763 100644 --- a/examples/asr_trainer.lua +++ b/examples/asr_trainer.lua @@ -4,7 +4,7 @@ function build_trainer(ifname) local layer_repo = make_layer_repo(sublayer_repo, param_repo) local crit = get_criterion_layer(sublayer_repo) local network = get_network(layer_repo) - local iterative_trainer = function (ofname, scp_file, bp) + local iterative_trainer = function (prefix, scp_file, bp) gconf.randomize = bp -- build buffer local buffer = make_buffer(make_reader(scp_file, layer_repo)) @@ -18,7 +18,7 @@ function build_trainer(ifname) print_stat(crit) gconf.cnt = 0 end - if gconf.cnt == 100 then break end +-- if gconf.cnt == 100 then break end input = {data.main_scp, data.phone_state} output = {} @@ -33,9 +33,10 @@ function build_trainer(ifname) collectgarbage("collect") end print_stat(crit) - if bp then + if (not bp) and prefix ~= nil then nerv.info("writing back...") - cf = nerv.ChunkFile(ofname, "w") + local accu_cv = get_accuracy(crit) + cf = nerv.ChunkFile(prefix .. "_cv" .. accu_cv .. ".nerv", "w") for i, p in ipairs(network:get_params()) do cf:write_chunk(p) end @@ -65,9 +66,10 @@ local do_halving = false nerv.info("initial cross validation: %.3f", accu_best) for i = 1, max_iter do nerv.info("iteration %d with lrate = %.6f", i, gconf.lrate) - local accu_tr = trainer(pf0 .. "_iter" .. i .. ".nerv", gconf.tr_scp, true) + local accu_tr = trainer(nil, gconf.tr_scp, true) nerv.info("[TR] training set %d: %.3f", i, accu_tr) - local accu_new = trainer(nil, gconf.cv_scp, false) + local accu_new = trainer(pf0 .. "_iter" .. i .. "_tr" .. accu_tr, + gconf.cv_scp, false) nerv.info("[CV] cross validation %d: %.3f", i, accu_new) -- TODO: revert the weights local accu_diff = accu_new - accu_best -- cgit v1.2.3-70-g09d2 From 6e720b961f7edac9c3a41affe0ca40dd0ec9fc85 Mon Sep 17 00:00:00 2001 From: Determinant Date: Sun, 7 Jun 2015 11:55:09 +0800 Subject: fix memory leak in profiling; other minor changes --- examples/asr_trainer.lua | 20 +++++++++++++------- examples/swb_baseline.lua | 14 +++++++------- matrix/cuda_helper.h | 13 +++++-------- matrix/cumatrix.c | 1 + matrix/generic/cumatrix.c | 2 ++ matrix/generic/mmatrix.c | 6 ------ nerv.lua | 4 +++- 7 files changed, 31 insertions(+), 29 deletions(-) (limited to 'examples') diff --git a/examples/asr_trainer.lua b/examples/asr_trainer.lua index d72b763..2993192 100644 --- a/examples/asr_trainer.lua +++ b/examples/asr_trainer.lua @@ -35,8 +35,9 @@ function build_trainer(ifname) print_stat(crit) if (not bp) and prefix ~= nil then nerv.info("writing back...") - local accu_cv = get_accuracy(crit) - cf = nerv.ChunkFile(prefix .. "_cv" .. accu_cv .. ".nerv", "w") + local fname = string.format("%s_cv%.3f.nerv", + prefix, get_accuracy(crit)) + cf = nerv.ChunkFile(fname, "w") for i, p in ipairs(network:get_params()) do cf:write_chunk(p) end @@ -53,7 +54,7 @@ halving_factor = 0.6 end_halving_inc = 0.1 min_iter = 1 max_iter = 20 -min_halving = 6 +min_halving = 5 gconf.batch_size = 256 gconf.buffer_size = 81920 @@ -65,11 +66,16 @@ local do_halving = false nerv.info("initial cross validation: %.3f", accu_best) for i = 1, max_iter do - nerv.info("iteration %d with lrate = %.6f", i, gconf.lrate) + nerv.info("[NN] begin iteration %d with lrate = %.6f", i, gconf.lrate) local accu_tr = trainer(nil, gconf.tr_scp, true) nerv.info("[TR] training set %d: %.3f", i, accu_tr) - local accu_new = trainer(pf0 .. "_iter" .. i .. "_tr" .. accu_tr, - gconf.cv_scp, false) + local accu_new = trainer( + string.format("%s_%s_iter_%d_lr%f_tr%.3f", + string.gsub(pf0, "(.*/)(.*)%..*", "%2"), + os.date("%Y%m%d%H%M%S"), + i, gconf.lrate, + accu_tr), + gconf.cv_scp, false) nerv.info("[CV] cross validation %d: %.3f", i, accu_new) -- TODO: revert the weights local accu_diff = accu_new - accu_best @@ -85,5 +91,5 @@ for i = 1, max_iter do if accu_new > accu_best then accu_best = accu_new end + nerv.Matrix.print_profile() end -nerv.Matrix.print_profile() diff --git a/examples/swb_baseline.lua b/examples/swb_baseline.lua index f536777..28cc6d5 100644 --- a/examples/swb_baseline.lua +++ b/examples/swb_baseline.lua @@ -6,8 +6,8 @@ gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9, tr_scp = "/slfs1/users/mfy43/swb_ivec/train_bp.scp", cv_scp = "/slfs1/users/mfy43/swb_ivec/train_cv.scp", htk_conf = "/slfs1/users/mfy43/swb_ivec/plp_0_d_a.conf", - global_transf = "global_transf.nerv", - initialized_param = "converted.nerv", + global_transf = "/slfs1/users/mfy43/swb_global_transf.nerv", + initialized_param = "/slfs1/users/mfy43/swb_init.nerv", debug = false} function make_param_repo(param_file) @@ -154,10 +154,10 @@ end function print_stat(crit) nerv.info("*** training stat begin ***") - nerv.utils.printf("cross entropy:\t%.8f\n", crit.total_ce) - nerv.utils.printf("correct:\t%d\n", crit.total_correct) - nerv.utils.printf("frames:\t%d\n", crit.total_frames) - nerv.utils.printf("err/frm:\t%.8f\n", crit.total_ce / crit.total_frames) - nerv.utils.printf("accuracy:\t%.3f%%\n", get_accuracy(crit)) + nerv.utils.printf("cross entropy:\t\t%.8f\n", crit.total_ce) + nerv.utils.printf("correct:\t\t%d\n", crit.total_correct) + nerv.utils.printf("frames:\t\t\t%d\n", crit.total_frames) + nerv.utils.printf("err/frm:\t\t%.8f\n", crit.total_ce / crit.total_frames) + nerv.utils.printf("accuracy:\t\t%.3f%%\n", get_accuracy(crit)) nerv.info("*** training stat end ***") end diff --git a/matrix/cuda_helper.h b/matrix/cuda_helper.h index d6effdb..fde6f18 100644 --- a/matrix/cuda_helper.h +++ b/matrix/cuda_helper.h @@ -62,17 +62,14 @@ static const char *cublasGetErrorString(cublasStatus_t err) { #define PROFILE_START \ do { \ - cudaEvent_t start, stop; \ - cudaEventCreate(&start); \ - cudaEventCreate(&stop); \ - cudaEventRecord(start, 0); + cudaEventRecord(profile_start, 0); #define PROFILE_STOP \ - cudaEventRecord(stop, 0); \ - cudaEventSynchronize(stop); \ + cudaEventRecord(profile_stop, 0); \ + cudaEventSynchronize(profile_stop); \ float milliseconds = 0; \ - cudaEventElapsedTime(&milliseconds, start, stop); \ + cudaEventElapsedTime(&milliseconds, profile_start, profile_stop); \ accu_profile(__func__, milliseconds / 1000); \ } while (0); -#define PROFILE_END +#define PROFILE_END #endif diff --git a/matrix/cumatrix.c b/matrix/cumatrix.c index 4ebc5ff..ee5ecaa 100644 --- a/matrix/cumatrix.c +++ b/matrix/cumatrix.c @@ -2,6 +2,7 @@ #include "../common.h" #include "cuda_helper.h" static cublasHandle_t cublas_handle; +static cudaEvent_t profile_start, profile_stop; static HashMap *profile; int print_profile(lua_State *L) { diff --git a/matrix/generic/cumatrix.c b/matrix/generic/cumatrix.c index 956e1e6..a340aef 100644 --- a/matrix/generic/cumatrix.c +++ b/matrix/generic/cumatrix.c @@ -443,6 +443,8 @@ static const luaL_Reg nerv_matrix_(extra_methods)[] = { static void cuda_matrix_(init)(lua_State *L) { luaN_append_methods(L, nerv_matrix_(extra_methods)); cublasCreate(&cublas_handle); + cudaEventCreate(&profile_start); + cudaEventCreate(&profile_stop); profile = hashmap_create(PROFILE_HASHMAP_SIZE, bkdr_hash, strcmp); } diff --git a/matrix/generic/mmatrix.c b/matrix/generic/mmatrix.c index 2045d65..b0f0791 100644 --- a/matrix/generic/mmatrix.c +++ b/matrix/generic/mmatrix.c @@ -87,15 +87,9 @@ int nerv_matrix_(save)(lua_State *L) { MATRIX_ELEM *row = MATRIX_ROW_PTR(self, i); for (j = 0; j < ncol; j++) if (fprintf(fp, MATRIX_ELEM_WRITE_FMT " ", row[j]) < 0) - { - free(self); return 0; - } if (fprintf(fp, "\n") < 0) - { - free(self); return 0; - } } return 0; } diff --git a/nerv.lua b/nerv.lua index ce6bc44..467d926 100644 --- a/nerv.lua +++ b/nerv.lua @@ -10,7 +10,9 @@ function nerv.error_method_not_implemented() end function nerv.info(fmt, ...) - nerv.utils.printf("[nerv] info: " .. fmt .. "\n", ...) + nerv.utils.printf( + string.format("(%s)[nerv] info: %s\n", + os.date("%H:%M:%S %F"), fmt), ...) end -- Torch C API wrapper -- cgit v1.2.3-70-g09d2 From 0f30b1a4b5e583cb1df7dbb349c1af4378e41369 Mon Sep 17 00:00:00 2001 From: Determinant Date: Sun, 7 Jun 2015 21:59:10 +0800 Subject: fix minor bugs in cumatrix; clean up part of code --- common.c | 2 +- examples/asr_trainer.lua | 7 +++++-- matrix/cukernel.cu | 1 + matrix/cumatrix.c | 28 ++++++++++++++++++++++++++-- matrix/generic/cumatrix.c | 6 ------ matrix/generic/matrix.c | 1 + matrix/init.c | 21 ++++++--------------- matrix/mmatrix.c | 13 +++++++++++++ 8 files changed, 53 insertions(+), 26 deletions(-) (limited to 'examples') diff --git a/common.c b/common.c index 355d7ff..b4e39e6 100644 --- a/common.c +++ b/common.c @@ -3,7 +3,7 @@ int nerv_error(lua_State *L, const char *err_mesg_fmt, ...) { va_list ap; va_start(ap, err_mesg_fmt); - lua_pushstring(L, "Nerv internal error: "); + lua_pushstring(L, "[nerv] internal error: "); lua_pushvfstring(L, err_mesg_fmt, ap); lua_concat(L, 2); lua_error(L); diff --git a/examples/asr_trainer.lua b/examples/asr_trainer.lua index 2993192..05d770f 100644 --- a/examples/asr_trainer.lua +++ b/examples/asr_trainer.lua @@ -33,6 +33,7 @@ function build_trainer(ifname) collectgarbage("collect") end print_stat(crit) + nerv.CuMatrix.print_profile() if (not bp) and prefix ~= nil then nerv.info("writing back...") local fname = string.format("%s_cv%.3f.nerv", @@ -71,7 +72,9 @@ for i = 1, max_iter do nerv.info("[TR] training set %d: %.3f", i, accu_tr) local accu_new = trainer( string.format("%s_%s_iter_%d_lr%f_tr%.3f", - string.gsub(pf0, "(.*/)(.*)%..*", "%2"), + string.gsub( + (string.gsub(pf0, "(.*/)(.*)", "%2")), + "(.*)%..*", "%1"), os.date("%Y%m%d%H%M%S"), i, gconf.lrate, accu_tr), @@ -91,5 +94,5 @@ for i = 1, max_iter do if accu_new > accu_best then accu_best = accu_new end - nerv.Matrix.print_profile() +-- nerv.Matrix.print_profile() end diff --git a/matrix/cukernel.cu b/matrix/cukernel.cu index fbac369..a19030a 100644 --- a/matrix/cukernel.cu +++ b/matrix/cukernel.cu @@ -9,6 +9,7 @@ #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR #undef MATRIX_ELEM_FMT +#undef MATRIX_ELEM_WRITE_FMT #define cudak_(NAME) cudak_double_ ## NAME #define MATRIX_USE_DOUBLE diff --git a/matrix/cumatrix.c b/matrix/cumatrix.c index ee5ecaa..af34fb4 100644 --- a/matrix/cumatrix.c +++ b/matrix/cumatrix.c @@ -1,11 +1,14 @@ #define NERV_GENERIC_CUMATRIX #include "../common.h" #include "cuda_helper.h" +#include +#define PROFILE_HASHMAP_SIZE 123457 static cublasHandle_t cublas_handle; static cudaEvent_t profile_start, profile_stop; static HashMap *profile; -int print_profile(lua_State *L) { +static int print_profile(lua_State *L) { + (void)L; size_t i; fprintf(stderr, "*** [nerv cumatrix profile] **\n"); for (i = 0; i < profile->size; i++) @@ -19,7 +22,8 @@ int print_profile(lua_State *L) { return 0; } -int clear_profile(lua_State *L) { +static int clear_profile(lua_State *L) { + (void)L; hashmap_clear(profile); return 0; } @@ -35,6 +39,25 @@ void accu_profile(const char *name, float delta) { *val += delta; } +static const luaL_Reg cumatrix_methods[] = { + {"print_profile", print_profile}, + {"clear_profile", clear_profile}, + {NULL, NULL} +}; + +extern void nerv_matrix_cuda_float_init(lua_State *L); +extern void nerv_matrix_cuda_double_init(lua_State *L); + +void nerv_cumatrix_init(lua_State *L) { + luaL_register(L, NULL, cumatrix_methods); + cublasCreate(&cublas_handle); + cudaEventCreate(&profile_start); + cudaEventCreate(&profile_stop); + profile = hashmap_create(PROFILE_HASHMAP_SIZE, bkdr_hash, strcmp); + nerv_matrix_cuda_float_init(L); + nerv_matrix_cuda_double_init(L); +} + #define MATRIX_USE_FLOAT #define cuda_matrix_(NAME) cuda_matrix_float_##NAME #define nerv_matrix_(NAME) nerv_matrix_cuda_float_##NAME @@ -51,6 +74,7 @@ const char *nerv_matrix_(tname) = "nerv.CuMatrixFloat"; #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR #undef MATRIX_ELEM_FMT +#undef MATRIX_ELEM_WRITE_FMT #undef MATRIX_CUMATRIX_HOST_TNAME #define MATRIX_USE_DOUBLE diff --git a/matrix/generic/cumatrix.c b/matrix/generic/cumatrix.c index a340aef..a8e18e0 100644 --- a/matrix/generic/cumatrix.c +++ b/matrix/generic/cumatrix.c @@ -11,11 +11,9 @@ #define MATRIX_BASE_TNAME nerv_matrix_cuda_tname #define NERV_GENERIC_MATRIX #define NERV_GENERIC_CUKERNEL -#define PROFILE_HASHMAP_SIZE 123457 #include "../../common.h" #include "../cukernel.h" #include "../cuda_helper.h" -#include Matrix *nerv_matrix_(new_)(lua_State *L, long nrow, long ncol); void nerv_matrix_(data_free)(lua_State *L, Matrix *self); @@ -442,10 +440,6 @@ static const luaL_Reg nerv_matrix_(extra_methods)[] = { static void cuda_matrix_(init)(lua_State *L) { luaN_append_methods(L, nerv_matrix_(extra_methods)); - cublasCreate(&cublas_handle); - cudaEventCreate(&profile_start); - cudaEventCreate(&profile_stop); - profile = hashmap_create(PROFILE_HASHMAP_SIZE, bkdr_hash, strcmp); } static void cuda_matrix_(free)(lua_State *L, MATRIX_ELEM *ptr) { diff --git a/matrix/generic/matrix.c b/matrix/generic/matrix.c index d6b0aea..e17fb42 100644 --- a/matrix/generic/matrix.c +++ b/matrix/generic/matrix.c @@ -6,6 +6,7 @@ extern const char *nerv_matrix_(tname); extern const char *MATRIX_BASE_TNAME; void nerv_matrix_(data_free)(lua_State *L, Matrix *self) { + (void)L; assert(*self->data_ref > 0); if (--(*self->data_ref) == 0) { diff --git a/matrix/init.c b/matrix/init.c index 7b7f478..c29d7e9 100644 --- a/matrix/init.c +++ b/matrix/init.c @@ -5,21 +5,14 @@ const char *nerv_matrix_tname = "nerv.Matrix"; const char *nerv_matrix_cuda_tname = "nerv.CuMatrix"; const char *nerv_matrix_host_tname = "nerv.MMatrix"; -void nerv_matrix_host_float_init(lua_State *L); -void nerv_matrix_cuda_float_init(lua_State *L); -void nerv_matrix_host_double_init(lua_State *L); -void nerv_matrix_cuda_double_init(lua_State *L); -void nerv_matrix_host_int_init(lua_State *L); -int print_profile(lua_State *L); -int clear_profile(lua_State *L); +void nerv_cumatrix_init(lua_State *L); +void nerv_mmatrix_init(lua_State *L); static const luaL_Reg matrix_methods[] = { {"__tostring__", nerv_error_method_not_implemented }, {"__add__", nerv_error_method_not_implemented }, {"__sub__", nerv_error_method_not_implemented }, {"__mul__", nerv_error_method_not_implemented }, - {"print_profile", print_profile}, - {"clear_profile", clear_profile}, {NULL, NULL} }; @@ -32,13 +25,11 @@ void nerv_matrix_init(lua_State *L) { /* CuMatrix inherits from Matrix */ luaT_newmetatable(L, nerv_matrix_cuda_tname, nerv_matrix_tname, NULL, NULL, NULL); - nerv_matrix_cuda_float_init(L); - nerv_matrix_cuda_double_init(L); - + nerv_cumatrix_init(L); + lua_pop(L, 1); /* MMatrix inherits from Matrix */ luaT_newmetatable(L, nerv_matrix_host_tname, nerv_matrix_tname, NULL, NULL, NULL); - nerv_matrix_host_float_init(L); - nerv_matrix_host_double_init(L); - nerv_matrix_host_int_init(L); + nerv_mmatrix_init(L); + lua_pop(L, 1); } diff --git a/matrix/mmatrix.c b/matrix/mmatrix.c index 81f8dfc..ffc058d 100644 --- a/matrix/mmatrix.c +++ b/matrix/mmatrix.c @@ -1,4 +1,15 @@ #define NERV_GENERIC_MMATRIX +#include "../common.h" +void nerv_matrix_host_float_init(lua_State *L); +void nerv_matrix_host_double_init(lua_State *L); +void nerv_matrix_host_int_init(lua_State *L); + +void nerv_mmatrix_init(lua_State *L) { + nerv_matrix_host_float_init(L); + nerv_matrix_host_double_init(L); + nerv_matrix_host_int_init(L); +} + #define MATRIX_USE_FLOAT #define host_matrix_(NAME) host_matrix_float_##NAME #define nerv_matrix_(NAME) nerv_matrix_host_float_##NAME @@ -10,6 +21,7 @@ const char *nerv_matrix_(tname) = "nerv.MMatrixFloat"; #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR #undef MATRIX_ELEM_FMT +#undef MATRIX_ELEM_WRITE_FMT #define NERV_GENERIC_MMATRIX #define MATRIX_USE_DOUBLE @@ -23,6 +35,7 @@ const char *nerv_matrix_(tname) = "nerv.MMatrixDouble"; #undef MATRIX_ELEM #undef MATRIX_ELEM_PTR #undef MATRIX_ELEM_FMT +#undef MATRIX_ELEM_WRITE_FMT #define NERV_GENERIC_MMATRIX #define MATRIX_USE_INT -- cgit v1.2.3-70-g09d2