aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDeterminant <[email protected]>2015-06-07 11:55:09 +0800
committerDeterminant <[email protected]>2015-06-07 11:55:09 +0800
commit6e720b961f7edac9c3a41affe0ca40dd0ec9fc85 (patch)
tree19ba0a682f9b75e70e18b796fbe5315ee5953d3a
parent5bcd5d79875587b08d598cc08bd5f8b1f5e14a23 (diff)
fix memory leak in profiling; other minor changes
-rw-r--r--examples/asr_trainer.lua20
-rw-r--r--examples/swb_baseline.lua14
-rw-r--r--matrix/cuda_helper.h13
-rw-r--r--matrix/cumatrix.c1
-rw-r--r--matrix/generic/cumatrix.c2
-rw-r--r--matrix/generic/mmatrix.c6
-rw-r--r--nerv.lua4
7 files changed, 31 insertions, 29 deletions
diff --git a/examples/asr_trainer.lua b/examples/asr_trainer.lua
index d72b763..2993192 100644
--- a/examples/asr_trainer.lua
+++ b/examples/asr_trainer.lua
@@ -35,8 +35,9 @@ function build_trainer(ifname)
print_stat(crit)
if (not bp) and prefix ~= nil then
nerv.info("writing back...")
- local accu_cv = get_accuracy(crit)
- cf = nerv.ChunkFile(prefix .. "_cv" .. accu_cv .. ".nerv", "w")
+ local fname = string.format("%s_cv%.3f.nerv",
+ prefix, get_accuracy(crit))
+ cf = nerv.ChunkFile(fname, "w")
for i, p in ipairs(network:get_params()) do
cf:write_chunk(p)
end
@@ -53,7 +54,7 @@ halving_factor = 0.6
end_halving_inc = 0.1
min_iter = 1
max_iter = 20
-min_halving = 6
+min_halving = 5
gconf.batch_size = 256
gconf.buffer_size = 81920
@@ -65,11 +66,16 @@ local do_halving = false
nerv.info("initial cross validation: %.3f", accu_best)
for i = 1, max_iter do
- nerv.info("iteration %d with lrate = %.6f", i, gconf.lrate)
+ nerv.info("[NN] begin iteration %d with lrate = %.6f", i, gconf.lrate)
local accu_tr = trainer(nil, gconf.tr_scp, true)
nerv.info("[TR] training set %d: %.3f", i, accu_tr)
- local accu_new = trainer(pf0 .. "_iter" .. i .. "_tr" .. accu_tr,
- gconf.cv_scp, false)
+ local accu_new = trainer(
+ string.format("%s_%s_iter_%d_lr%f_tr%.3f",
+ string.gsub(pf0, "(.*/)(.*)%..*", "%2"),
+ os.date("%Y%m%d%H%M%S"),
+ i, gconf.lrate,
+ accu_tr),
+ gconf.cv_scp, false)
nerv.info("[CV] cross validation %d: %.3f", i, accu_new)
-- TODO: revert the weights
local accu_diff = accu_new - accu_best
@@ -85,5 +91,5 @@ for i = 1, max_iter do
if accu_new > accu_best then
accu_best = accu_new
end
+ nerv.Matrix.print_profile()
end
-nerv.Matrix.print_profile()
diff --git a/examples/swb_baseline.lua b/examples/swb_baseline.lua
index f536777..28cc6d5 100644
--- a/examples/swb_baseline.lua
+++ b/examples/swb_baseline.lua
@@ -6,8 +6,8 @@ gconf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9,
tr_scp = "/slfs1/users/mfy43/swb_ivec/train_bp.scp",
cv_scp = "/slfs1/users/mfy43/swb_ivec/train_cv.scp",
htk_conf = "/slfs1/users/mfy43/swb_ivec/plp_0_d_a.conf",
- global_transf = "global_transf.nerv",
- initialized_param = "converted.nerv",
+ global_transf = "/slfs1/users/mfy43/swb_global_transf.nerv",
+ initialized_param = "/slfs1/users/mfy43/swb_init.nerv",
debug = false}
function make_param_repo(param_file)
@@ -154,10 +154,10 @@ end
function print_stat(crit)
nerv.info("*** training stat begin ***")
- nerv.utils.printf("cross entropy:\t%.8f\n", crit.total_ce)
- nerv.utils.printf("correct:\t%d\n", crit.total_correct)
- nerv.utils.printf("frames:\t%d\n", crit.total_frames)
- nerv.utils.printf("err/frm:\t%.8f\n", crit.total_ce / crit.total_frames)
- nerv.utils.printf("accuracy:\t%.3f%%\n", get_accuracy(crit))
+ nerv.utils.printf("cross entropy:\t\t%.8f\n", crit.total_ce)
+ nerv.utils.printf("correct:\t\t%d\n", crit.total_correct)
+ nerv.utils.printf("frames:\t\t\t%d\n", crit.total_frames)
+ nerv.utils.printf("err/frm:\t\t%.8f\n", crit.total_ce / crit.total_frames)
+ nerv.utils.printf("accuracy:\t\t%.3f%%\n", get_accuracy(crit))
nerv.info("*** training stat end ***")
end
diff --git a/matrix/cuda_helper.h b/matrix/cuda_helper.h
index d6effdb..fde6f18 100644
--- a/matrix/cuda_helper.h
+++ b/matrix/cuda_helper.h
@@ -62,17 +62,14 @@ static const char *cublasGetErrorString(cublasStatus_t err) {
#define PROFILE_START \
do { \
- cudaEvent_t start, stop; \
- cudaEventCreate(&start); \
- cudaEventCreate(&stop); \
- cudaEventRecord(start, 0);
+ cudaEventRecord(profile_start, 0);
#define PROFILE_STOP \
- cudaEventRecord(stop, 0); \
- cudaEventSynchronize(stop); \
+ cudaEventRecord(profile_stop, 0); \
+ cudaEventSynchronize(profile_stop); \
float milliseconds = 0; \
- cudaEventElapsedTime(&milliseconds, start, stop); \
+ cudaEventElapsedTime(&milliseconds, profile_start, profile_stop); \
accu_profile(__func__, milliseconds / 1000); \
} while (0);
-#define PROFILE_END
+#define PROFILE_END
#endif
diff --git a/matrix/cumatrix.c b/matrix/cumatrix.c
index 4ebc5ff..ee5ecaa 100644
--- a/matrix/cumatrix.c
+++ b/matrix/cumatrix.c
@@ -2,6 +2,7 @@
#include "../common.h"
#include "cuda_helper.h"
static cublasHandle_t cublas_handle;
+static cudaEvent_t profile_start, profile_stop;
static HashMap *profile;
int print_profile(lua_State *L) {
diff --git a/matrix/generic/cumatrix.c b/matrix/generic/cumatrix.c
index 956e1e6..a340aef 100644
--- a/matrix/generic/cumatrix.c
+++ b/matrix/generic/cumatrix.c
@@ -443,6 +443,8 @@ static const luaL_Reg nerv_matrix_(extra_methods)[] = {
static void cuda_matrix_(init)(lua_State *L) {
luaN_append_methods(L, nerv_matrix_(extra_methods));
cublasCreate(&cublas_handle);
+ cudaEventCreate(&profile_start);
+ cudaEventCreate(&profile_stop);
profile = hashmap_create(PROFILE_HASHMAP_SIZE, bkdr_hash, strcmp);
}
diff --git a/matrix/generic/mmatrix.c b/matrix/generic/mmatrix.c
index 2045d65..b0f0791 100644
--- a/matrix/generic/mmatrix.c
+++ b/matrix/generic/mmatrix.c
@@ -87,15 +87,9 @@ int nerv_matrix_(save)(lua_State *L) {
MATRIX_ELEM *row = MATRIX_ROW_PTR(self, i);
for (j = 0; j < ncol; j++)
if (fprintf(fp, MATRIX_ELEM_WRITE_FMT " ", row[j]) < 0)
- {
- free(self);
return 0;
- }
if (fprintf(fp, "\n") < 0)
- {
- free(self);
return 0;
- }
}
return 0;
}
diff --git a/nerv.lua b/nerv.lua
index ce6bc44..467d926 100644
--- a/nerv.lua
+++ b/nerv.lua
@@ -10,7 +10,9 @@ function nerv.error_method_not_implemented()
end
function nerv.info(fmt, ...)
- nerv.utils.printf("[nerv] info: " .. fmt .. "\n", ...)
+ nerv.utils.printf(
+ string.format("(%s)[nerv] info: %s\n",
+ os.date("%H:%M:%S %F"), fmt), ...)
end
-- Torch C API wrapper