From 5edaa7d5649a2e124496159f7e8a14edab4f7484 Mon Sep 17 00:00:00 2001 From: Determinant Date: Sat, 30 May 2015 20:52:05 +0800 Subject: rename param file to chunk file (intend to generalize it) --- Makefile | 3 +- examples/test_dnn_layers.lua | 13 +- io/chunk_file.c | 305 +++++++++++++++++++++++++++++++++++++++++++ io/chunk_file.h | 22 ++++ io/init.c | 6 +- io/init.lua | 26 ++-- io/param.c | 305 ------------------------------------------- io/param.h | 22 ---- layer/affine.lua | 3 +- layer/init.lua | 13 +- matrix/generic/cumatrix.c | 1 - matrix/generic/mmatrix.c | 8 +- nerv.c | 4 +- 13 files changed, 367 insertions(+), 364 deletions(-) create mode 100644 io/chunk_file.c create mode 100644 io/chunk_file.h delete mode 100644 io/param.c delete mode 100644 io/param.h diff --git a/Makefile b/Makefile index b4ba715..69fb739 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ BUILD_DIR := $(CURDIR)/build OBJS := nerv.o luaT.o common.o \ matrix/mmatrix.o matrix/cumatrix.o matrix/init.o matrix/cukernel.o \ - io/init.o io/param.o \ + io/init.o io/chunk_file.o \ examples/oop_example.o LIBS := libnerv.so LUA_LIBS := matrix/init.lua io/init.lua nerv.lua \ @@ -56,3 +56,4 @@ speech: clean: -rm -rf $(OBJ_DIR) -rm -rf $(LUA_DIR) + -rm -rf $(LIB_DIR) diff --git a/examples/test_dnn_layers.lua b/examples/test_dnn_layers.lua index c57de6d..866e685 100644 --- a/examples/test_dnn_layers.lua +++ b/examples/test_dnn_layers.lua @@ -2,11 +2,12 @@ require 'layer.affine' require 'layer.sigmoid' require 'layer.softmax_ce' -global_conf = {lrate = 0.8, wcost = 1e-6, momentum = 0.9} +global_conf = {lrate = 0.8, wcost = 1e-6, + momentum = 0.9, mat_type = nerv.CuMatrixFloat} -pf = nerv.ParamFile("affine.param", "r") -ltp = pf:read_param("a") -bp = pf:read_param("b") +pf = nerv.ChunkFile("affine.param", "r") +ltp = pf:read_chunk("a", global_conf) +bp = pf:read_chunk("b", global_conf) -- print(bp.trans) @@ -18,7 +19,7 @@ af:init() sg:init() sm:init() -df = nerv.ParamFile("input.param", "r") +df = nerv.ChunkFile("input.param", "r") label = nerv.CuMatrixFloat(10, 2048) label:fill(0) @@ -26,7 +27,7 @@ for i = 0, 9 do label[i][i] = 1.0 end -input1 = {[0] = df:read_param("input").trans} +input1 = {[0] = df:read_chunk("input", global_conf).trans} output1 = {[0] = nerv.CuMatrixFloat(10, 2048)} input2 = output1 output2 = {[0] = nerv.CuMatrixFloat(10, 2048)} diff --git a/io/chunk_file.c b/io/chunk_file.c new file mode 100644 index 0000000..ce346c5 --- /dev/null +++ b/io/chunk_file.c @@ -0,0 +1,305 @@ +#include +#include +#include +#include "../common.h" +#include "chunk_file.h" + +#define INVALID_FORMAT_ERROR(fn) \ + nerv_error(L, "Invalid chunk file: %s", fn) +#define CHECK_FORMAT(exp, ret, fname) \ + do { \ + if ((exp) != (ret)) INVALID_FORMAT_ERROR(fn); \ + } while (0) + +const char *nerv_chunk_file_tname = "nerv.ChunkFile"; +const char *nerv_chunk_file_handle_tname = "nerv.ChunkFileHandle"; +const char *nerv_chunk_info_tname = "nerv.ChunkInfo"; +const char *nerv_chunk_data_tname = "nerv.ChunkData"; + +#define PARAM_HEADER_SIZE 16 + +enum { + NORMAL, + INVALID_FORMAT, + END_OF_FILE, + SECTION_OVERFLOW, + WRITE_ERROR +}; + +size_t read_chunk_header_plain(FILE *fp, int *status) { + static char buff[PARAM_HEADER_SIZE]; + int i; + size_t size = 0; + *status = NORMAL; + if (fread(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE) + { + if (feof(fp)) *status = END_OF_FILE; + else *status = INVALID_FORMAT; + } + for (i = 0; i < PARAM_HEADER_SIZE; i++) + if (isdigit(buff[i])) + size = size * 10 + buff[i] - '0'; + fprintf(stderr, "header: %lu\n", size); + return size; +} + +#define CHECK_WRITE(status) \ + do { \ + if (status == SECTION_OVERFLOW) \ + nerv_error(L, "section overflowed"); \ + else if (status == WRITE_ERROR) \ + nerv_error(L, "error while writing"); \ + } while (0) + +void write_chunk_header_plain(FILE *fp, size_t size, int *status) { + static char buff[PARAM_HEADER_SIZE]; + int i; + *status = NORMAL; + for (i = PARAM_HEADER_SIZE - 3; i > 0; i--, size /= 10) + buff[i] = size % 10 + '0'; + if (size) + { + *status = SECTION_OVERFLOW; + return; + } + buff[0] = '['; + buff[PARAM_HEADER_SIZE - 2] = ']'; + buff[PARAM_HEADER_SIZE - 1] = '\n'; + if (fwrite(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE) + { + *status = WRITE_ERROR; + return; + } +} + +ChunkData *get_chunk_data(FILE *fp, ChunkInfo *info) { + ChunkData *pcd = (ChunkData *)malloc(sizeof(ChunkData)); + pcd->data = (char *)malloc(info->length); + pcd->fp = fmemopen(pcd->data, info->length, "r"); + assert(fseeko(fp, info->offset, SEEK_SET) == 0); + if (fread(pcd->data, 1, info->length, fp) != (size_t)info->length) + return NULL; + return pcd; +} + +const char *read_chunk_metadata(lua_State *L, FILE *fp, const char *fn) { +#define LINEBUFF_SIZE 1024 + static char buff[7 + LINEBUFF_SIZE] = "return "; + CHECK_FORMAT(fgets(buff + 7, LINEBUFF_SIZE, fp), buff + 7, fn); + fprintf(stderr, "metadata: %s\n", buff); + return buff; +} + +void write_chunk_metadata(FILE *fp, const char *metadata_str, int *status) { + size_t size = strlen(metadata_str); + *status = NORMAL; + if (fwrite(metadata_str, 1, size, fp) != size || + fprintf(fp, "\n") < 0) + { + *status = WRITE_ERROR; + return; + } + fprintf(stderr, "metadata: %s\n", metadata_str); +} + + +int nerv_chunk_file_open_write(lua_State *L, const char *fn) { + FILE *fp = fopen(fn, "w"); + ChunkFileHandle *lfp; + if (!fp) nerv_error(L, "Error while opening chunk file: %s", fn); + lfp = (ChunkFileHandle *)malloc(sizeof(ChunkFileHandle)); + lfp->fp = fp; + luaT_pushudata(L, lfp, nerv_chunk_file_handle_tname); + lua_setfield(L, -2, "handle"); + luaT_pushmetatable(L, nerv_chunk_file_tname); + lua_setmetatable(L, -2); + return 1; +} + +int nerv_chunk_file_open_read(lua_State *L, const char *fn) { + FILE *fp = fopen(fn, "r"); + int i, status; + size_t chunk_len; + off_t offset; + ChunkFileHandle *lfp; + + if (!fp) nerv_error(L, "Error while opening chunk file: %s", fn); + offset = ftello(fp); + lua_newtable(L); + fprintf(stderr, "%d\n", (int)offset); + for (i = 0;; offset += chunk_len, i++) + { + ChunkInfo *pci; + fprintf(stderr, "reading chunk %d from %d\n", i, (int)offset); + /* skip to the begining of chunk i */ + CHECK_FORMAT(fseeko(fp, offset, SEEK_SET), 0, fn); + /* read header */ + chunk_len = read_chunk_header_plain(fp, &status); + if (status == END_OF_FILE) break; + else if (status == INVALID_FORMAT) + INVALID_FORMAT_ERROR(fn); + /* read metadata */ + luaL_loadstring(L, read_chunk_metadata(L, fp, fn)); + CHECK_FORMAT(lua_pcall(L, 0, 1, 0), 0, fn); + CHECK_FORMAT(lua_istable(L, -1), 1, fn); + /* stack: obj_table, metadata */ + /* chunk info */ + pci = (ChunkInfo *)malloc(sizeof(ChunkInfo)); + pci->offset = ftello(fp); + pci->length = chunk_len - (pci->offset - offset); + fprintf(stderr, "%d + %d (skip %lu)\n", (int)pci->offset, + (int)pci->length, chunk_len); + luaT_pushudata(L, pci, nerv_chunk_info_tname); + lua_setfield(L, -2, "chunk"); + /* stack: obj_table, metadata */ + /* get id */ + lua_getfield(L, -1, "id"); + /* stack: obj_table, metadata, id */ + if (!lua_isstring(L, -1)) + nerv_error(L, "id field in metadata must be a string"); + lua_pushvalue(L, -1); + /* stack: obj_table, metadata, id, id */ + lua_gettable(L, -4); + /* stack: obj_table, metadata, id, obj[id] */ + if (!lua_isnil(L, -1)) + nerv_error(L, "conflicting id"); + lua_pop(L, 1); + /* stack: obj_table, metadata, id */ + lua_pushvalue(L, -2); + /* stack: obj_table, metadata, id, metadata */ + lua_settable(L, -4); + /* stack: obj_table, metadata */ + lua_pop(L, 1); + } + lua_setfield(L, -2, "metadata"); + lfp = (ChunkFileHandle *)malloc(sizeof(ChunkFileHandle)); + lfp->fp = fp; + luaT_pushudata(L, lfp, nerv_chunk_file_handle_tname); + lua_setfield(L, -2, "handle"); + luaT_pushmetatable(L, nerv_chunk_file_tname); + lua_setmetatable(L, -2); + return 1; +} + +int nerv_chunk_file_new_(lua_State *L, const char *fn, const char *mode) { + int rd = 1, bin = 0; + size_t i, len = strlen(mode); + for (i = 0; i < len; i++) + switch (mode[i]) + { + case 'r': rd = 1; break; + case 'w': rd = 0; break; + case 'b': bin = 1; break; + } + return rd ? nerv_chunk_file_open_read(L, fn) : \ + nerv_chunk_file_open_write(L, fn); +} + +int nerv_chunk_file___init(lua_State *L) { + lua_pushvalue(L, 1); + return nerv_chunk_file_new_(L, luaL_checkstring(L, 2), + luaL_checkstring(L, 3)); +} + +int nerv_chunk_file_new(lua_State *L) { + lua_newtable(L); + return nerv_chunk_file_new_(L, luaL_checkstring(L, 1), + luaL_checkstring(L, 2)); +} + +int nerv_chunk_file_write_chunkdata(lua_State *L) { + ChunkFileHandle *pfh; + int status; + off_t start; + size_t size; + const char *metadata_str = lua_tolstring(L, 2, NULL); + lua_getfield(L, 1, "handle"); + pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname); + start = ftello(pfh->fp); + write_chunk_header_plain(pfh->fp, 0, &status); /* fill zeros */ + CHECK_WRITE(status); + write_chunk_metadata(pfh->fp, metadata_str, &status); + CHECK_WRITE(status); + lua_pushvalue(L, 3); + lua_getfield(L, -1, "write"); + if (!lua_isfunction(L, -1)) + nerv_error(L, "\"write\" method must be implemented"); + lua_pushvalue(L, -2); + lua_pushvalue(L, 4); /* pass handle as parameter to write() */ + lua_call(L, 2, 0); /* let the write() to write */ + lua_pop(L, 1); + size = ftello(pfh->fp) - start; + fseeko(pfh->fp, start, SEEK_SET); + /* write the calced size */ + write_chunk_header_plain(pfh->fp, size, &status); + CHECK_WRITE(status); + fseeko(pfh->fp, 0, SEEK_END); + return 0; +} + +int nerv_chunk_file_get_chunkdata(lua_State *L) { + ChunkFileHandle *pfh; + ChunkInfo *pci; + ChunkData *pcd; + const char *id = luaL_checkstring(L, 2); + + lua_getfield(L, 1, "handle"); + pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname); + lua_pop(L, 1); /* pop handle */ + lua_getfield(L, 1, "metadata"); + /* now stack: self, k, metadata */ + lua_getfield(L, -1, id); + /* now stack: self, k, metadata, kth{} */ + if (lua_isnil(L, -1)) /* no chunck with the id */ + return 0; + lua_getfield(L, -1, "chunk"); + pci = luaT_checkudata(L, -1, nerv_chunk_info_tname); + if (!(pcd = get_chunk_data(pfh->fp, pci))) + nerv_error(L, "unexpected end of file"); + luaT_pushudata(L, pcd, nerv_chunk_data_tname); + return 1; +} + +int nerv_chunk_file_handle_destroy(lua_State *L) { + ChunkFileHandle *pfh = luaT_checkudata(L, 1, + nerv_chunk_file_handle_tname); + fclose(pfh->fp); + free(pfh); + return 0; +} + +static int nerv_chunk_destroy(lua_State *L) { + ChunkInfo *pci = luaT_checkudata(L, 1, nerv_chunk_info_tname); + free(pci); + return 0; +} + +static int nerv_chunk_data_destroy(lua_State *L) { + ChunkData *pcd = luaT_checkudata(L, 1, nerv_chunk_data_tname); + fclose(pcd->fp); + free(pcd->data); + free(pcd); + return 0; +} + +static const luaL_Reg nerv_chunk_file_methods[] = { + {"get_chunkdata", nerv_chunk_file_get_chunkdata}, + {"_write_chunkdata", nerv_chunk_file_write_chunkdata}, + {"__init", nerv_chunk_file___init}, + {NULL, NULL} +}; + +void nerv_chunk_file_init(lua_State *L) { + luaT_newmetatable(L, nerv_chunk_file_tname, NULL, + nerv_chunk_file_new, + NULL, NULL); + luaL_register(L, NULL, nerv_chunk_file_methods); + lua_pop(L, 1); + luaT_newmetatable(L, nerv_chunk_file_handle_tname, NULL, + NULL, nerv_chunk_file_handle_destroy, NULL); + luaT_newmetatable(L, nerv_chunk_info_tname, NULL, + NULL, nerv_chunk_destroy, NULL); + luaT_newmetatable(L, nerv_chunk_data_tname, NULL, + NULL, nerv_chunk_data_destroy, NULL); +} + diff --git a/io/chunk_file.h b/io/chunk_file.h new file mode 100644 index 0000000..9ece117 --- /dev/null +++ b/io/chunk_file.h @@ -0,0 +1,22 @@ +#ifndef NERV_LAYER_FILE_H +#define NERV_LAYER_FILE_H + +extern const char *nerv_chunk_file_tname; +extern const char *nerv_chunk_file_handle_tname; +extern const char *nerv_chunk_info_tname; +extern const char *nerv_chunk_data_tname; + +typedef struct ChunkFileHandle { + FILE *fp; +} ChunkFileHandle; + +typedef struct ChunkInfo { + off_t offset, length; +} ChunkInfo; + +typedef struct ChunkData { + FILE *fp; + char *data; +} ChunkData; + +#endif diff --git a/io/init.c b/io/init.c index d299d54..70585f7 100644 --- a/io/init.c +++ b/io/init.c @@ -1,6 +1,6 @@ #include "../common.h" -extern void nerv_param_file_init(lua_State *L); -void nerv_param_init(lua_State *L) { - nerv_param_file_init(L); +extern void nerv_chunk_file_init(lua_State *L); +void nerv_io_init(lua_State *L) { + nerv_chunk_file_init(L); } diff --git a/io/init.lua b/io/init.lua index dc1c6c3..8c3701e 100644 --- a/io/init.lua +++ b/io/init.lua @@ -1,4 +1,4 @@ -function nerv.ParamFile:write_chunkdata(metadata, writer) +function nerv.ChunkFile:write_chunkdata(metadata, writer) if type(metadata) ~= "table" then nerv.error("metadata should be a Lua table") return @@ -6,25 +6,25 @@ function nerv.ParamFile:write_chunkdata(metadata, writer) return self:_write_chunkdata(table.tostring(metadata), writer) end -function nerv.ParamFile:write_param(param) - local id = param.id - local type = param.__typename +function nerv.ChunkFile:write_chunk(chunk) + local id = chunk.id + local type = chunk.__typename if id == nil then - nerv_error("id of param %s must be specified", type) + nerv_error("id of chunk %s must be specified", type) end self:write_chunkdata({id = id, type = type, - info = param:get_info()}, param) + info = chunk:get_info()}, chunk) end -function nerv.ParamFile:read_param(id) +function nerv.ChunkFile:read_chunk(id, global_conf) local metadata = self.metadata[id] if metadata == nil then - nerv_error("param with id %s does not exist", id) + nerv_error("chunk with id %s does not exist", id) end - local param = assert(loadstring("return " .. - metadata.type .. "(\"" .. id .. "\")"))() - param:set_info(metadata.info) - param:read(self:get_chunkdata(id)) - return param + local chunk_type = assert(loadstring("return " .. metadata.type))() + local chunk = chunk_type(id, global_conf) + chunk:set_info(metadata.info) + chunk:read(self:get_chunkdata(id)) + return chunk end diff --git a/io/param.c b/io/param.c deleted file mode 100644 index a57b183..0000000 --- a/io/param.c +++ /dev/null @@ -1,305 +0,0 @@ -#include -#include -#include -#include "../common.h" -#include "param.h" - -#define INVALID_FORMAT_ERROR(fn) \ - nerv_error(L, "Invalid param file: %s", fn) -#define CHECK_FORMAT(exp, ret, fname) \ - do { \ - if ((exp) != (ret)) INVALID_FORMAT_ERROR(fn); \ - } while (0) - -const char *nerv_param_file_tname = "nerv.ParamFile"; -const char *nerv_param_file_handle_tname = "nerv.ParamFileHandle"; -const char *nerv_param_chunk_info_tname = "nerv.ParamChunkInfo"; -const char *nerv_param_chunk_data_tname = "nerv.ParamChunkData"; - -#define PARAM_HEADER_SIZE 16 - -enum { - NORMAL, - INVALID_FORMAT, - END_OF_FILE, - SECTION_OVERFLOW, - WRITE_ERROR -}; - -size_t read_param_header_plain(FILE *fp, int *status) { - static char buff[PARAM_HEADER_SIZE]; - int i; - size_t size = 0; - *status = NORMAL; - if (fread(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE) - { - if (feof(fp)) *status = END_OF_FILE; - else *status = INVALID_FORMAT; - } - for (i = 0; i < PARAM_HEADER_SIZE; i++) - if (isdigit(buff[i])) - size = size * 10 + buff[i] - '0'; - fprintf(stderr, "header: %lu\n", size); - return size; -} - -#define CHECK_WRITE(status) \ - do { \ - if (status == SECTION_OVERFLOW) \ - nerv_error(L, "section overflowed"); \ - else if (status == WRITE_ERROR) \ - nerv_error(L, "error while writing"); \ - } while (0) - -void write_param_header_plain(FILE *fp, size_t size, int *status) { - static char buff[PARAM_HEADER_SIZE]; - int i; - *status = NORMAL; - for (i = PARAM_HEADER_SIZE - 3; i > 0; i--, size /= 10) - buff[i] = size % 10 + '0'; - if (size) - { - *status = SECTION_OVERFLOW; - return; - } - buff[0] = '['; - buff[PARAM_HEADER_SIZE - 2] = ']'; - buff[PARAM_HEADER_SIZE - 1] = '\n'; - if (fwrite(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE) - { - *status = WRITE_ERROR; - return; - } -} - -ParamChunkData *get_param_chunk_data(FILE *fp, ParamChunkInfo *info) { - ParamChunkData *pcd = (ParamChunkData *)malloc(sizeof(ParamChunkData)); - pcd->data = (char *)malloc(info->length); - pcd->fp = fmemopen(pcd->data, info->length, "r"); - assert(fseeko(fp, info->offset, SEEK_SET) == 0); - if (fread(pcd->data, 1, info->length, fp) != (size_t)info->length) - return NULL; - return pcd; -} - -const char *read_param_metadata(lua_State *L, FILE *fp, const char *fn) { -#define LINEBUFF_SIZE 1024 - static char buff[7 + LINEBUFF_SIZE] = "return "; - CHECK_FORMAT(fgets(buff + 7, LINEBUFF_SIZE, fp), buff + 7, fn); - fprintf(stderr, "metadata: %s\n", buff); - return buff; -} - -void write_param_metadata(FILE *fp, const char *metadata_str, int *status) { - size_t size = strlen(metadata_str); - *status = NORMAL; - if (fwrite(metadata_str, 1, size, fp) != size || - fprintf(fp, "\n") < 0) - { - *status = WRITE_ERROR; - return; - } - fprintf(stderr, "metadata: %s\n", metadata_str); -} - - -int nerv_param_file_open_write(lua_State *L, const char *fn) { - FILE *fp = fopen(fn, "w"); - ParamFileHandle *lfp; - if (!fp) nerv_error(L, "Error while opening param file: %s", fn); - lfp = (ParamFileHandle *)malloc(sizeof(ParamFileHandle)); - lfp->fp = fp; - luaT_pushudata(L, lfp, nerv_param_file_handle_tname); - lua_setfield(L, -2, "handle"); - luaT_pushmetatable(L, nerv_param_file_tname); - lua_setmetatable(L, -2); - return 1; -} - -int nerv_param_file_open_read(lua_State *L, const char *fn) { - FILE *fp = fopen(fn, "r"); - int i, status; - size_t param_len; - off_t offset; - ParamFileHandle *lfp; - - if (!fp) nerv_error(L, "Error while opening param file: %s", fn); - offset = ftello(fp); - lua_newtable(L); - fprintf(stderr, "%d\n", (int)offset); - for (i = 0;; offset += param_len, i++) - { - ParamChunkInfo *pci; - fprintf(stderr, "reading param chunk %d from %d\n", i, (int)offset); - /* skip to the begining of param chunk i */ - CHECK_FORMAT(fseeko(fp, offset, SEEK_SET), 0, fn); - /* read header */ - param_len = read_param_header_plain(fp, &status); - if (status == END_OF_FILE) break; - else if (status == INVALID_FORMAT) - INVALID_FORMAT_ERROR(fn); - /* read metadata */ - luaL_loadstring(L, read_param_metadata(L, fp, fn)); - CHECK_FORMAT(lua_pcall(L, 0, 1, 0), 0, fn); - CHECK_FORMAT(lua_istable(L, -1), 1, fn); - /* stack: obj_table, metadata */ - /* chunk info */ - pci = (ParamChunkInfo *)malloc(sizeof(ParamChunkInfo)); - pci->offset = ftello(fp); - pci->length = param_len - (pci->offset - offset); - fprintf(stderr, "%d + %d (skip %lu)\n", (int)pci->offset, - (int)pci->length, param_len); - luaT_pushudata(L, pci, nerv_param_chunk_info_tname); - lua_setfield(L, -2, "chunk"); - /* stack: obj_table, metadata */ - /* get id */ - lua_getfield(L, -1, "id"); - /* stack: obj_table, metadata, id */ - if (!lua_isstring(L, -1)) - nerv_error(L, "id field in metadata must be a string"); - lua_pushvalue(L, -1); - /* stack: obj_table, metadata, id, id */ - lua_gettable(L, -4); - /* stack: obj_table, metadata, id, obj[id] */ - if (!lua_isnil(L, -1)) - nerv_error(L, "conflicting id"); - lua_pop(L, 1); - /* stack: obj_table, metadata, id */ - lua_pushvalue(L, -2); - /* stack: obj_table, metadata, id, metadata */ - lua_settable(L, -4); - /* stack: obj_table, metadata */ - lua_pop(L, 1); - } - lua_setfield(L, -2, "metadata"); - lfp = (ParamFileHandle *)malloc(sizeof(ParamFileHandle)); - lfp->fp = fp; - luaT_pushudata(L, lfp, nerv_param_file_handle_tname); - lua_setfield(L, -2, "handle"); - luaT_pushmetatable(L, nerv_param_file_tname); - lua_setmetatable(L, -2); - return 1; -} - -int nerv_param_file_new_(lua_State *L, const char *fn, const char *mode) { - int rd = 1, bin = 0; - size_t i, len = strlen(mode); - for (i = 0; i < len; i++) - switch (mode[i]) - { - case 'r': rd = 1; break; - case 'w': rd = 0; break; - case 'b': bin = 1; break; - } - return rd ? nerv_param_file_open_read(L, fn) : \ - nerv_param_file_open_write(L, fn); -} - -int nerv_param_file___init(lua_State *L) { - lua_pushvalue(L, 1); - return nerv_param_file_new_(L, luaL_checkstring(L, 2), - luaL_checkstring(L, 3)); -} - -int nerv_param_file_new(lua_State *L) { - lua_newtable(L); - return nerv_param_file_new_(L, luaL_checkstring(L, 1), - luaL_checkstring(L, 2)); -} - -int nerv_param_file_write_chunkdata(lua_State *L) { - ParamFileHandle *pfh; - int status; - off_t start; - size_t size; - const char *metadata_str = lua_tolstring(L, 2, NULL); - lua_getfield(L, 1, "handle"); - pfh = luaT_checkudata(L, -1, nerv_param_file_handle_tname); - start = ftello(pfh->fp); - write_param_header_plain(pfh->fp, 0, &status); /* fill zeros */ - CHECK_WRITE(status); - write_param_metadata(pfh->fp, metadata_str, &status); - CHECK_WRITE(status); - lua_pushvalue(L, 3); - lua_getfield(L, -1, "write"); - if (!lua_isfunction(L, -1)) - nerv_error(L, "\"write\" method must be implemented"); - lua_pushvalue(L, -2); - lua_pushvalue(L, 4); /* pass handle as parameter to write() */ - lua_call(L, 2, 0); /* let the write() to write */ - lua_pop(L, 1); - size = ftello(pfh->fp) - start; - fseeko(pfh->fp, start, SEEK_SET); - /* write the calced size */ - write_param_header_plain(pfh->fp, size, &status); - CHECK_WRITE(status); - fseeko(pfh->fp, 0, SEEK_END); - return 0; -} - -int nerv_param_file_get_chunkdata(lua_State *L) { - ParamFileHandle *pfh; - ParamChunkInfo *pci; - ParamChunkData *pcd; - const char *id = luaL_checkstring(L, 2); - - lua_getfield(L, 1, "handle"); - pfh = luaT_checkudata(L, -1, nerv_param_file_handle_tname); - lua_pop(L, 1); /* pop handle */ - lua_getfield(L, 1, "metadata"); - /* now stack: self, k, metadata */ - lua_getfield(L, -1, id); - /* now stack: self, k, metadata, kth{} */ - if (lua_isnil(L, -1)) /* no chunck with the id */ - return 0; - lua_getfield(L, -1, "chunk"); - pci = luaT_checkudata(L, -1, nerv_param_chunk_info_tname); - if (!(pcd = get_param_chunk_data(pfh->fp, pci))) - nerv_error(L, "unexpected end of file"); - luaT_pushudata(L, pcd, nerv_param_chunk_data_tname); - return 1; -} - -int nerv_param_file_handle_destroy(lua_State *L) { - ParamFileHandle *pfh = luaT_checkudata(L, 1, - nerv_param_file_handle_tname); - fclose(pfh->fp); - free(pfh); - return 0; -} - -static int nerv_param_chunk_destroy(lua_State *L) { - ParamChunkInfo *pci = luaT_checkudata(L, 1, nerv_param_chunk_info_tname); - free(pci); - return 0; -} - -static int nerv_param_chunk_data_destroy(lua_State *L) { - ParamChunkData *pcd = luaT_checkudata(L, 1, nerv_param_chunk_data_tname); - fclose(pcd->fp); - free(pcd->data); - free(pcd); - return 0; -} - -static const luaL_Reg nerv_param_file_methods[] = { - {"get_chunkdata", nerv_param_file_get_chunkdata}, - {"_write_chunkdata", nerv_param_file_write_chunkdata}, - {"__init", nerv_param_file___init}, - {NULL, NULL} -}; - -void nerv_param_file_init(lua_State *L) { - luaT_newmetatable(L, nerv_param_file_tname, NULL, - nerv_param_file_new, - NULL, NULL); - luaL_register(L, NULL, nerv_param_file_methods); - lua_pop(L, 1); - luaT_newmetatable(L, nerv_param_file_handle_tname, NULL, - NULL, nerv_param_file_handle_destroy, NULL); - luaT_newmetatable(L, nerv_param_chunk_info_tname, NULL, - NULL, nerv_param_chunk_destroy, NULL); - luaT_newmetatable(L, nerv_param_chunk_data_tname, NULL, - NULL, nerv_param_chunk_data_destroy, NULL); -} - diff --git a/io/param.h b/io/param.h deleted file mode 100644 index e5841b8..0000000 --- a/io/param.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef NERV_LAYER_FILE_H -#define NERV_LAYER_FILE_H - -extern const char *nerv_param_file_tname; -extern const char *nerv_param_file_handle_tname; -extern const char *nerv_param_chunk_info_tname; -extern const char *nerv_param_chunk_data_tname; - -typedef struct ParamFileHandle { - FILE *fp; -} ParamFileHandle; - -typedef struct ParamChunkInfo { - off_t offset, length; -} ParamChunkInfo; - -typedef struct ParamChunkData { - FILE *fp; - char *data; -} ParamChunkData; - -#endif diff --git a/layer/affine.lua b/layer/affine.lua index 97703a8..d88813e 100644 --- a/layer/affine.lua +++ b/layer/affine.lua @@ -3,7 +3,8 @@ local BiasParam = nerv.class('nerv.BiasParam', 'nerv.LinearTransParam') local AffineLayer = nerv.class('nerv.AffineLayer', 'nerv.Layer') function LinearTransParam:read(pcdata) - self.trans = nerv.CuMatrixFloat.new_from_host(nerv.MMatrixFloat.load(pcdata)) + self.trans = self.gconf.mat_type.new_from_host( + nerv.MMatrixFloat.load(pcdata)) end function LinearTransParam:write(pfhandle) diff --git a/layer/init.lua b/layer/init.lua index 0f0afe8..a98621d 100644 --- a/layer/init.lua +++ b/layer/init.lua @@ -2,12 +2,9 @@ local Param = nerv.class('nerv.Param') -function nerv.Param:__init(id) +function nerv.Param:__init(id, global_conf) self.id = id -end - -function nerv.Param:init(id) - nerv.error_method_not_implemented() + self.gconf = global_conf end function nerv.Param:get_info() @@ -28,7 +25,11 @@ end local Layer = nerv.class('nerv.Layer') -function nerv.Layer:_init(id, global_conf, ...) +function nerv.Layer:__init(id, global_conf, ...) + nerv.error_method_not_implemented() +end + +function nerv.Layer:init(id) nerv.error_method_not_implemented() end diff --git a/matrix/generic/cumatrix.c b/matrix/generic/cumatrix.c index ed64bbf..c4ba937 100644 --- a/matrix/generic/cumatrix.c +++ b/matrix/generic/cumatrix.c @@ -90,7 +90,6 @@ static int nerv_matrix_(mul)(lua_State *L) { static int nerv_matrix_(create)(lua_State *L) { Matrix *a = luaT_checkudata(L, 1, nerv_matrix_(tname)); - fprintf(stderr, "create\n"); Matrix *b = nerv_matrix_(new_)(a->nrow, a->ncol); luaT_pushudata(L, b, nerv_matrix_(tname)); return 1; diff --git a/matrix/generic/mmatrix.c b/matrix/generic/mmatrix.c index c25b9f7..4b43572 100644 --- a/matrix/generic/mmatrix.c +++ b/matrix/generic/mmatrix.c @@ -11,7 +11,7 @@ #define MATRIX_BASE_TNAME nerv_matrix_host_tname #define NERV_GENERIC_MATRIX #include "../../common.h" -#include "../../io/param.h" +#include "../../io/chunk_file.h" static void host_matrix_(alloc)(MATRIX_ELEM **dptr, size_t *stride, long width, long height) { @@ -46,7 +46,7 @@ static void host_matrix_(init)(lua_State *L) { #include "matrix.c" int nerv_matrix_(load)(lua_State *L) { - ParamChunkData *chunk = luaT_checkudata(L, 1, nerv_param_chunk_data_tname); + ChunkData *chunk = luaT_checkudata(L, 1, nerv_chunk_data_tname); Matrix *self; int i, j; long nrow, ncol; @@ -69,8 +69,8 @@ int nerv_matrix_(load)(lua_State *L) { } int nerv_matrix_(save)(lua_State *L) { - ParamFileHandle *chunk = luaT_checkudata(L, 2, - nerv_param_file_handle_tname); + ChunkFileHandle *chunk = luaT_checkudata(L, 2, + nerv_chunk_file_handle_tname); Matrix *self = luaT_checkudata(L, 1, nerv_matrix_(tname)); int i, j; long nrow = self->nrow, ncol = self->ncol; diff --git a/nerv.c b/nerv.c index fa47c62..a59eadc 100644 --- a/nerv.c +++ b/nerv.c @@ -2,7 +2,7 @@ extern void nerv_example_init(lua_State *L); extern void nerv_matrix_init(lua_State *L); -extern void nerv_param_init(lua_State *L); +extern void nerv_io_init(lua_State *L); static const luaL_Reg nerv_utils_methods[] = { {"setmetatable", luaT_lua_setmetatable}, @@ -33,6 +33,6 @@ int luaopen_libnerv(lua_State *L) { nerv_utils_init(L); nerv_example_init(L); nerv_matrix_init(L); - nerv_param_init(L); + nerv_io_init(L); return 1; } -- cgit v1.2.3