diff options
| -rw-r--r-- | nerv/Makefile | 8 | ||||
| -rw-r--r-- | nerv/common.h | 2 | ||||
| -rw-r--r-- | nerv/io/chunk_file.c | 345 | ||||
| -rw-r--r-- | nerv/io/chunk_file.h | 19 | ||||
| -rw-r--r-- | nerv/io/init.lua | 8 | ||||
| -rw-r--r-- | nerv/lib/io/chunk_file.c | 239 | ||||
| -rw-r--r-- | nerv/lib/io/chunk_file.h | 43 | ||||
| -rw-r--r-- | nerv/lib/luaT/README.md (renamed from nerv/luaT/README.md) | 0 | ||||
| -rw-r--r-- | nerv/lib/luaT/luaT.c (renamed from nerv/luaT/luaT.c) | 0 | ||||
| -rw-r--r-- | nerv/lib/luaT/luaT.h (renamed from nerv/luaT/luaT.h) | 0 | ||||
| -rw-r--r-- | nerv/matrix/generic/mmatrix.c | 10 | ||||
| -rw-r--r-- | nerv/nerv-scm-1.rockspec | 10 |
12 files changed, 385 insertions, 299 deletions
diff --git a/nerv/Makefile b/nerv/Makefile index b69a63e..4008453 100644 --- a/nerv/Makefile +++ b/nerv/Makefile @@ -1,7 +1,7 @@ .PHONY: build install clean SHELL := /bin/bash BUILD_DIR := $(CURDIR)/build -OBJS := nerv.o luaT.o common.o \ +OBJS := nerv.o lib/luaT/luaT.o lib/io/chunk_file.o common.o \ matrix/mmatrix.o matrix/cumatrix.o matrix/init.o matrix/cukernel.o \ io/init.o io/chunk_file.o \ examples/oop_example.o @@ -19,7 +19,7 @@ INCLUDE += $(CUDA_INCLUDE) LDFLAGS := -L$(CUDA_BASE)/lib64/ -Wl,-rpath=$(CUDA_BASE)/lib64/ -lcudart -lcublas CFLAGS := -Wall -Wextra -O2 OBJ_DIR := $(BUILD_DIR)/objs -SUBDIR := matrix io layer examples nn +SUBDIR := matrix io layer examples nn lib/io lib/luaT NVCC := $(CUDA_BASE)/bin/nvcc NVCC_FLAGS := -Xcompiler -fPIC,-Wall,-Wextra @@ -39,8 +39,8 @@ $(OBJ_DIR)/matrix/cukernel.o: matrix/cukernel.cu $(NVCC) -c -o $@ $< $(INCLUDE) $(NVCC_FLAGS) $(LUA_DIR)/%.lua: %.lua cp $< $@ -$(OBJ_DIR)/luaT.o: - gcc -c -o $@ luaT/luaT.c $(INCLUDE) -fPIC +#$(OBJ_DIR)/luaT.o: +# gcc -c -o $@ luaT/luaT.c $(INCLUDE) -fPIC $(LIBS): $(OBJS) gcc -shared -o $@ $(OBJS) $(LDFLAGS) diff --git a/nerv/common.h b/nerv/common.h index e21c7a5..6657dc4 100644 --- a/nerv/common.h +++ b/nerv/common.h @@ -3,7 +3,7 @@ #include "lua.h" #include "lauxlib.h" #include "lualib.h" -#include "luaT/luaT.h" +#include "lib/luaT/luaT.h" #include <stdio.h> #include <stdlib.h> diff --git a/nerv/io/chunk_file.c b/nerv/io/chunk_file.c index c0b6b9f..e275d9b 100644 --- a/nerv/io/chunk_file.c +++ b/nerv/io/chunk_file.c @@ -5,321 +5,136 @@ #include "chunk_file.h" #define INVALID_FORMAT_ERROR(fn) \ - nerv_error(L, "Invalid chunk file: %s", fn) + nerv_error(L, "invalid chunk file: %s", fn) #define CHECK_FORMAT(exp, ret, fname) \ do { \ if ((exp) != (ret)) INVALID_FORMAT_ERROR(fn); \ } while (0) -#define CHECK_FILE_OPEN(pfh) \ - do { \ - if ((pfh)->closed) \ - nerv_error(L, "operations on a closed file"); \ - } while (0) const char *nerv_chunk_file_tname = "nerv.ChunkFile"; const char *nerv_chunk_file_handle_tname = "nerv.ChunkFileHandle"; const char *nerv_chunk_info_tname = "nerv.ChunkInfo"; const char *nerv_chunk_data_tname = "nerv.ChunkData"; -#define PARAM_HEADER_SIZE 16 - -enum { - NORMAL, - INVALID_FORMAT, - END_OF_FILE, - SECTION_OVERFLOW, - WRITE_ERROR -}; - -size_t read_chunk_header_plain(FILE *fp, int *status) { - static char buff[PARAM_HEADER_SIZE]; - int i; - size_t size = 0; - *status = NORMAL; - if (fread(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE) - { - if (feof(fp)) *status = END_OF_FILE; - else *status = INVALID_FORMAT; - } - for (i = 0; i < PARAM_HEADER_SIZE; i++) - if (isdigit(buff[i])) - size = size * 10 + buff[i] - '0'; -/* fprintf(stderr, "header: %lu\n", size); */ - return size; -} - -#define CHECK_WRITE(status) \ - do { \ - if (status == SECTION_OVERFLOW) \ - nerv_error(L, "section overflowed"); \ - else if (status == WRITE_ERROR) \ - nerv_error(L, "error while writing"); \ - } while (0) - -void write_chunk_header_plain(FILE *fp, size_t size, int *status) { - static char buff[PARAM_HEADER_SIZE]; - int i; - *status = NORMAL; - for (i = PARAM_HEADER_SIZE - 3; i > 0; i--, size /= 10) - buff[i] = size % 10 + '0'; - if (size) - { - *status = SECTION_OVERFLOW; - return; - } - buff[0] = '['; - buff[PARAM_HEADER_SIZE - 2] = ']'; - buff[PARAM_HEADER_SIZE - 1] = '\n'; - if (fwrite(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE) - { - *status = WRITE_ERROR; - return; - } -} - -ChunkData *get_chunk_data(FILE *fp, ChunkInfo *info) { - ChunkData *pcd = (ChunkData *)malloc(sizeof(ChunkData)); - pcd->data = (char *)malloc(info->length); - pcd->fp = fmemopen(pcd->data, info->length, "r"); - assert(fseeko(fp, info->offset, SEEK_SET) == 0); - if (fread(pcd->data, 1, info->length, fp) != (size_t)info->length) - return NULL; - return pcd; -} - -const char *read_chunk_metadata(lua_State *L, FILE *fp, const char *fn) { -#define LINEBUFF_SIZE 1024 - static char buff[7 + LINEBUFF_SIZE] = "return "; - CHECK_FORMAT(fgets(buff + 7, LINEBUFF_SIZE, fp), buff + 7, fn); - /* fprintf(stderr, "metadata: %s\n", buff); */ - return buff; -} - -void write_chunk_metadata(FILE *fp, const char *metadata_str, int *status) { - size_t size = strlen(metadata_str); - *status = NORMAL; - if (fwrite(metadata_str, 1, size, fp) != size || - fprintf(fp, "\n") < 0) +int nerv_lua_chunk_file_new(lua_State *L) { + int status; + const char *fn = luaL_checkstring(L, 1); + ChunkFile *cfp = nerv_chunk_file_create(fn, + luaL_checkstring(L, 2), + &status); + if (status != CF_NORMAL) { - *status = WRITE_ERROR; - return; + nerv_error(L, "%s: %s", fn, nerv_chunk_file_errstr(status)); } - /* fprintf(stderr, "metadata: %s\n", metadata_str); */ -} - - -int nerv_chunk_file_open_write(lua_State *L, const char *fn) { - FILE *fp = fopen(fn, "w"); - ChunkFileHandle *lfp; - if (!fp) nerv_error(L, "Error while opening chunk file: %s", fn); - lfp = (ChunkFileHandle *)malloc(sizeof(ChunkFileHandle)); - lfp->fp = fp; - lfp->closed = 0; - luaT_pushudata(L, lfp, nerv_chunk_file_handle_tname); - lua_setfield(L, -2, "handle"); - luaT_pushmetatable(L, nerv_chunk_file_tname); - lua_setmetatable(L, -2); - return 1; -} - -int nerv_chunk_file_open_read(lua_State *L, const char *fn) { - FILE *fp = fopen(fn, "r"); - int i, status; - size_t chunk_len; - off_t offset; - ChunkFileHandle *lfp; - - if (!fp) nerv_error(L, "Error while opening chunk file: %s", fn); - offset = ftello(fp); lua_newtable(L); - /* fprintf(stderr, "%d\n", (int)offset); */ - for (i = 0;; offset += chunk_len, i++) + luaT_pushudata(L, cfp, nerv_chunk_file_handle_tname); + lua_setfield(L, -2, "handle"); + if (cfp->status == CF_READ) { - ChunkInfo *pci; - /* fprintf(stderr, "reading chunk %d from %d\n", i, (int)offset); */ - /* skip to the begining of chunk i */ - CHECK_FORMAT(fseeko(fp, offset, SEEK_SET), 0, fn); - /* read header */ - chunk_len = read_chunk_header_plain(fp, &status); - if (status == END_OF_FILE) break; - else if (status == INVALID_FORMAT) - INVALID_FORMAT_ERROR(fn); - /* read metadata */ - luaL_loadstring(L, read_chunk_metadata(L, fp, fn)); - CHECK_FORMAT(lua_pcall(L, 0, 1, 0), 0, fn); - CHECK_FORMAT(lua_istable(L, -1), 1, fn); - /* stack: obj_table, metadata */ - /* chunk info */ - pci = (ChunkInfo *)malloc(sizeof(ChunkInfo)); - pci->offset = ftello(fp); - pci->length = chunk_len - (pci->offset - offset); - /* fprintf(stderr, "%d + %d (skip %lu)\n", (int)pci->offset, - (int)pci->length, chunk_len); */ - luaT_pushudata(L, pci, nerv_chunk_info_tname); - lua_setfield(L, -2, "chunk"); - /* stack: obj_table, metadata */ - /* get id */ - lua_getfield(L, -1, "id"); - /* stack: obj_table, metadata, id */ - if (!lua_isstring(L, -1)) - nerv_error(L, "id field in metadata must be a string"); - lua_pushvalue(L, -1); - /* stack: obj_table, metadata, id, id */ - lua_gettable(L, -4); - /* stack: obj_table, metadata, id, obj[id] */ - if (!lua_isnil(L, -1)) - nerv_error(L, "conflicting id"); - lua_pop(L, 1); - /* stack: obj_table, metadata, id */ - lua_pushvalue(L, -2); - /* stack: obj_table, metadata, id, metadata */ - lua_settable(L, -4); - /* stack: obj_table, metadata */ - lua_pop(L, 1); + ChunkInfo *cip; + /* build a table with interpreted metadata in Lua because C API only + * provides with linked list with uninterpreted metadata */ + lua_newtable(L); + /* stack: self, metadata_table */ + for (cip = cfp->info; cip; cip = cip->next) + { + luaL_loadstring(L, cip->metadata); + CHECK_FORMAT(lua_pcall(L, 0, 1, 0), 0, fn); + CHECK_FORMAT(lua_istable(L, -1), 1, fn); + lua_getfield(L, -1, "id"); + if (!lua_isstring(L, -1)) + nerv_error(L, "id field in metadata must be a string"); + /* stack: ... metadata_table, metadata, id */ + lua_pushvalue(L, -1); + /* stack: ... metadata_table, metadata, id, id */ + lua_gettable(L, -4); + /* stack: ... metadata_table, metadata, id, metadata_table[id] */ + if (!lua_isnil(L, -1)) + nerv_error(L, "conflicting id"); + lua_pop(L, 1); + /* stack: ... metadata_table, metadata, id */ + lua_pushvalue(L, -2); + /* stack: ... metadata_table, metadata, id, metadata */ + lua_settable(L, -4); + /* stack: ... metadata_table, metadata */ + luaT_pushudata(L, cip, nerv_chunk_info_tname); + /* stack: ... metadata_table, cip */ + lua_setfield(L, -2, "_chunk_info"); + /* stack: ... metadata_table */ + lua_pop(L, 1); + /* stack: ... metadata_table */ + } + lua_setfield(L, -2, "metadata"); + /* stack: ... */ } - lua_setfield(L, -2, "metadata"); - lfp = (ChunkFileHandle *)malloc(sizeof(ChunkFileHandle)); - lfp->fp = fp; - lfp->closed = 0; - luaT_pushudata(L, lfp, nerv_chunk_file_handle_tname); - lua_setfield(L, -2, "handle"); luaT_pushmetatable(L, nerv_chunk_file_tname); lua_setmetatable(L, -2); return 1; } -int nerv_chunk_file_new_(lua_State *L, const char *fn, const char *mode) { - int rd = 1, bin = 0; - size_t i, len = strlen(mode); - for (i = 0; i < len; i++) - switch (mode[i]) - { - case 'r': rd = 1; break; - case 'w': rd = 0; break; - case 'b': bin = 1; break; - } - return rd ? nerv_chunk_file_open_read(L, fn) : \ - nerv_chunk_file_open_write(L, fn); +static void writer(void *L) { + lua_call((lua_State *)L, 2, 0); /* let the write() to write */ } -int nerv_chunk_file___init(lua_State *L) { - lua_pushvalue(L, 1); - return nerv_chunk_file_new_(L, luaL_checkstring(L, 2), - luaL_checkstring(L, 3)); -} - -int nerv_chunk_file_new(lua_State *L) { - lua_newtable(L); - return nerv_chunk_file_new_(L, luaL_checkstring(L, 1), - luaL_checkstring(L, 2)); -} - -int nerv_chunk_file_write_chunkdata(lua_State *L) { - ChunkFileHandle *pfh; +int nerv_lua_chunk_file_write_chunkdata(lua_State *L) { int status; - off_t start; - size_t size; - const char *metadata_str = lua_tolstring(L, 2, NULL); - lua_getfield(L, 1, "handle"); - pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname); - CHECK_FILE_OPEN(pfh); - start = ftello(pfh->fp); - write_chunk_header_plain(pfh->fp, 0, &status); /* fill zeros */ - CHECK_WRITE(status); - write_chunk_metadata(pfh->fp, metadata_str, &status); - CHECK_WRITE(status); - lua_pushvalue(L, 3); - lua_getfield(L, -1, "write"); + ChunkFile *cfp = luaT_checkudata(L, 1, nerv_chunk_file_handle_tname); + const char *mdstr = lua_tolstring(L, 2, NULL); + lua_getfield(L, 3, "write"); if (!lua_isfunction(L, -1)) nerv_error(L, "\"write\" method must be implemented"); - lua_pushvalue(L, -2); - lua_pushvalue(L, 4); /* pass handle as parameter to write() */ - lua_call(L, 2, 0); /* let the write() to write */ - lua_pop(L, 1); - size = ftello(pfh->fp) - start; - fseeko(pfh->fp, start, SEEK_SET); - /* write the calced size */ - write_chunk_header_plain(pfh->fp, size, &status); - CHECK_WRITE(status); - fseeko(pfh->fp, 0, SEEK_END); + lua_pushvalue(L, 3); /* lua writer itself */ + lua_pushvalue(L, 1); /* pass handle as parameter to write() */ + nerv_chunk_file_write_chunkdata(cfp, mdstr, writer, (void *)L); return 0; } -int nerv_chunk_file_get_chunkdata(lua_State *L) { - ChunkFileHandle *pfh; - ChunkInfo *pci; - ChunkData *pcd; - const char *id = luaL_checkstring(L, 2); - - lua_getfield(L, 1, "handle"); - pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname); - CHECK_FILE_OPEN(pfh); - lua_pop(L, 1); /* pop handle */ - lua_getfield(L, 1, "metadata"); - /* now stack: self, k, metadata */ - lua_getfield(L, -1, id); - /* now stack: self, k, metadata, kth{} */ - if (lua_isnil(L, -1)) /* no chunck with the id */ - return 0; - lua_getfield(L, -1, "chunk"); - pci = luaT_checkudata(L, -1, nerv_chunk_info_tname); - if (!(pcd = get_chunk_data(pfh->fp, pci))) - nerv_error(L, "unexpected end of file"); - luaT_pushudata(L, pcd, nerv_chunk_data_tname); +int nerv_lua_chunk_file_get_chunkdata(lua_State *L) { + int status; + ChunkFile *cfp = luaT_checkudata(L, 1, nerv_chunk_file_handle_tname); + ChunkInfo *cip = luaT_checkudata(L, 2, nerv_chunk_info_tname); + ChunkData *cdp = nerv_chunk_file_get_chunkdata(cfp, cip, &status); + if (status != CF_NORMAL) + nerv_error(L, "%s", nerv_chunk_file_errstr(status)); + luaT_pushudata(L, cdp, nerv_chunk_data_tname); return 1; } -int nerv_chunk_file_close(lua_State *L) { - ChunkFileHandle *pfh; - lua_getfield(L, 1, "handle"); - pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname); - CHECK_FILE_OPEN(pfh); - fclose(pfh->fp); - pfh->closed = 1; - return 0; -} - -int nerv_chunk_file_handle_destroy(lua_State *L) { - ChunkFileHandle *pfh = luaT_checkudata(L, 1, - nerv_chunk_file_handle_tname); - if (!pfh->closed) fclose(pfh->fp); - free(pfh); +int nerv_lua_chunk_file_close(lua_State *L) { + ChunkFile *cfp = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname); + nerv_chunk_file_close(cfp); return 0; } -static int nerv_chunk_info_destroy(lua_State *L) { - ChunkInfo *pci = luaT_checkudata(L, 1, nerv_chunk_info_tname); - free(pci); +int nerv_lua_chunk_file_destroy(lua_State *L) { + ChunkFile *cfp = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname); + nerv_chunk_file_destroy(cfp); return 0; } -static int nerv_chunk_data_destroy(lua_State *L) { - ChunkData *pcd = luaT_checkudata(L, 1, nerv_chunk_data_tname); - fclose(pcd->fp); - free(pcd->data); - free(pcd); +static int nerv_lua_chunk_data_destroy(lua_State *L) { + ChunkData *cdp = luaT_checkudata(L, 1, nerv_chunk_data_tname); + nerv_chunk_data_destroy(cdp); return 0; } static const luaL_Reg nerv_chunk_file_methods[] = { - {"get_chunkdata", nerv_chunk_file_get_chunkdata}, - {"_write_chunkdata", nerv_chunk_file_write_chunkdata}, - {"close", nerv_chunk_file_close}, - {"__init", nerv_chunk_file___init}, + {"_get_chunkdata", nerv_lua_chunk_file_get_chunkdata}, + {"_write_chunkdata", nerv_lua_chunk_file_write_chunkdata}, + {"_close", nerv_lua_chunk_file_close}, {NULL, NULL} }; void nerv_chunk_file_init(lua_State *L) { luaT_newmetatable(L, nerv_chunk_file_tname, NULL, - nerv_chunk_file_new, + nerv_lua_chunk_file_new, NULL, NULL); luaL_register(L, NULL, nerv_chunk_file_methods); lua_pop(L, 1); luaT_newmetatable(L, nerv_chunk_file_handle_tname, NULL, - NULL, nerv_chunk_file_handle_destroy, NULL); + NULL, nerv_lua_chunk_file_destroy, NULL); luaT_newmetatable(L, nerv_chunk_info_tname, NULL, - NULL, nerv_chunk_info_destroy, NULL); + NULL, NULL, NULL); luaT_newmetatable(L, nerv_chunk_data_tname, NULL, - NULL, nerv_chunk_data_destroy, NULL); + NULL, nerv_lua_chunk_data_destroy, NULL); } - diff --git a/nerv/io/chunk_file.h b/nerv/io/chunk_file.h index 9bae59d..71094a5 100644 --- a/nerv/io/chunk_file.h +++ b/nerv/io/chunk_file.h @@ -1,23 +1,10 @@ -#ifndef NERV_LAYER_FILE_H -#define NERV_LAYER_FILE_H +#ifndef NERV_LUA_CHUNK_FILE_H +#define NERV_LUA_CHUNK_FILE_H +#include "../lib/io/chunk_file.h" extern const char *nerv_chunk_file_tname; extern const char *nerv_chunk_file_handle_tname; extern const char *nerv_chunk_info_tname; extern const char *nerv_chunk_data_tname; -typedef struct ChunkFileHandle { - FILE *fp; - int closed; -} ChunkFileHandle; - -typedef struct ChunkInfo { - off_t offset, length; -} ChunkInfo; - -typedef struct ChunkData { - FILE *fp; - char *data; -} ChunkData; - #endif diff --git a/nerv/io/init.lua b/nerv/io/init.lua index 647ff93..eb2e3e5 100644 --- a/nerv/io/init.lua +++ b/nerv/io/init.lua @@ -3,7 +3,7 @@ function nerv.ChunkFile:write_chunkdata(metadata, writer) nerv.error("metadata should be a Lua table") return end - return self:_write_chunkdata(table.tostring(metadata), writer) + return self._write_chunkdata(self.handle, table.tostring(metadata), writer) end function nerv.ChunkFile:write_chunk(chunk) @@ -28,10 +28,14 @@ function nerv.ChunkFile:read_chunk(id, global_conf) local chunk_type = nerv.get_type(metadata.type) local chunk = chunk_type(id, global_conf) chunk:set_info(metadata.info) - chunk:read(self:get_chunkdata(id)) + chunk:read(self._get_chunkdata(self.handle, metadata._chunk_info)) return chunk end +function nerv.ChunkFile:close() + self._close(self.handle) +end + local DataReader = nerv.class("nerv.DataReader") function DataReader:__init(global_conf, reader_conf) diff --git a/nerv/lib/io/chunk_file.c b/nerv/lib/io/chunk_file.c new file mode 100644 index 0000000..a305962 --- /dev/null +++ b/nerv/lib/io/chunk_file.c @@ -0,0 +1,239 @@ +#include "../../common.h" +#include "chunk_file.h" +#include <stdlib.h> +#include <string.h> +#define PARAM_HEADER_SIZE 16 + +static size_t read_chunk_header_plain(FILE *fp, int *status) { + static char buff[PARAM_HEADER_SIZE]; + int i; + size_t size = 0; + if (fread(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE) + { + if (feof(fp)) *status = CF_END_OF_FILE; + else + { + *status = CF_INVALID_FORMAT; + return 0; + } + } + else *status = CF_NORMAL; + for (i = 0; i < PARAM_HEADER_SIZE; i++) + if (isdigit(buff[i])) + size = size * 10 + buff[i] - '0'; +/* fprintf(stderr, "header: %lu\n", size); */ + return size; +} + +static void write_chunk_header_plain(FILE *fp, size_t size, int *status) { + static char buff[PARAM_HEADER_SIZE]; + int i; + for (i = PARAM_HEADER_SIZE - 3; i > 0; i--, size /= 10) + buff[i] = size % 10 + '0'; + if (size) + { + *status = CF_SECTION_OVERFLOW; + return; + } + buff[0] = '['; + buff[PARAM_HEADER_SIZE - 2] = ']'; + buff[PARAM_HEADER_SIZE - 1] = '\n'; + if (fwrite(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE) + { + *status = CF_WRITE_ERROR; + return; + } + *status = CF_NORMAL; +} + +static ChunkData *get_chunk_data(FILE *fp, ChunkInfo *info) { + ChunkData *cdp = (ChunkData *)malloc(sizeof(ChunkData)); + cdp->data = (char *)malloc(info->length); + cdp->fp = fmemopen(cdp->data, info->length, "r"); + assert(fseeko(fp, info->offset, SEEK_SET) == 0); + if (fread(cdp->data, 1, info->length, fp) != (size_t)info->length) + return NULL; + return cdp; +} + +static const char *read_chunk_metadata(FILE *fp, const char *fn, int *status) { +#define LINEBUFF_SIZE 1024 +#define LUA_RETURN "return " +#define LUA_RETURN_LEN (sizeof(LUA_RETURN) - 1) + static char buff[LUA_RETURN_LEN + LINEBUFF_SIZE] = LUA_RETURN; + *status = fgets(buff + LUA_RETURN_LEN, + LINEBUFF_SIZE, fp) == (buff + LUA_RETURN_LEN) ? \ + CF_NORMAL : CF_INVALID_FORMAT; + fprintf(stderr, "metadata: %s\n", buff); + return buff; +} + +static void write_chunk_metadata(FILE *fp, const char *mdstr, int *status) { + size_t size = strlen(mdstr); + if (fwrite(mdstr, 1, size, fp) != size || + fprintf(fp, "\n") < 0) + { + *status = CF_WRITE_ERROR; + return; + } + /* fprintf(stderr, "metadata: %s\n", metadata_str); */ + *status = CF_NORMAL; +} + +static ChunkFile *open_write(const char *fn, int *status) { + ChunkFile *cfp; + FILE *fp = fopen(fn, "w"); + + if (!fp) + { + *status = CF_ERR_OPEN_FILE; + return NULL; + } + cfp = (ChunkFile *)malloc(sizeof(ChunkFile)); + cfp->fp = fp; + cfp->status = CF_WRITE; + *status = CF_NORMAL; + return cfp; +} + +static ChunkFile *open_read(const char *fn, int *status) { + size_t chunk_len; + off_t offset; + int i; + const char *mdstr; + ChunkFile *cfp; + ChunkInfo *head = NULL; + FILE *fp = fopen(fn, "r"); + + if (!fp) + { + *status = CF_ERR_OPEN_FILE; + return NULL; + } + cfp = (ChunkFile *)malloc(sizeof(ChunkFile)); + cfp->fp = fp; + cfp->status = CF_READ; + offset = ftello(fp); + /* fprintf(stderr, "%d\n", (int)offset); */ + for (i = 0;; offset += chunk_len, i++) + { + ChunkInfo *cip; + fprintf(stderr, "reading chunk %d from %d\n", i, (int)offset); + /* skip to the begining of chunk i */ + if (fseeko(fp, offset, SEEK_SET) != 0) + { + *status = CF_INVALID_FORMAT; + return NULL; + } + /* read header */ + chunk_len = read_chunk_header_plain(fp, status); + if (*status == CF_END_OF_FILE) break; + if (*status != CF_NORMAL) + return NULL; + cip = (ChunkInfo *)malloc(sizeof(ChunkInfo)); + /* read metadata */ + mdstr = read_chunk_metadata(fp, fn, status); + if (*status != CF_NORMAL) + return NULL; + cip->metadata = strdup(mdstr); + cip->offset = ftello(fp); + cip->length = chunk_len - (cip->offset - offset); + /* fprintf(stderr, "%d + %d (skip %lu)\n", (int)cip->offset, + (int)cip->length, chunk_len); */ + cip->next = head; + head = cip; + } + *status = CF_NORMAL; + cfp->info = head; + return cfp; +} + +ChunkFile *nerv_chunk_file_create(const char *fn, const char *mode, int *status) { + int rd = 1, bin = 0; + size_t i, len = strlen(mode); + for (i = 0; i < len; i++) + switch (mode[i]) + { + case 'r': rd = 1; break; + case 'w': rd = 0; break; + case 'b': bin = 1; break; + } + return rd ? open_read(fn, status) : \ + open_write(fn, status); +} + +int nerv_chunk_file_write_chunkdata(ChunkFile *cfp, const char *mdstr, + ChunkDataWriter_t writer, void *writer_arg) { + int status; + off_t start; + size_t size; + if (cfp->status != CF_WRITE) + return CF_INVALID_OP; + start = ftello(cfp->fp); + write_chunk_header_plain(cfp->fp, 0, &status); /* fill zeros */ + if (status != CF_NORMAL) return status; + write_chunk_metadata(cfp->fp, mdstr, &status); + if (status != CF_NORMAL) return status; + writer(writer_arg); + size = ftello(cfp->fp) - start; + fseeko(cfp->fp, start, SEEK_SET); + /* write the calced size */ + write_chunk_header_plain(cfp->fp, size, &status); + if (status != CF_NORMAL) return status; + fseeko(cfp->fp, 0, SEEK_END); + return CF_NORMAL; +} + +ChunkData *nerv_chunk_file_get_chunkdata(ChunkFile *cfp, ChunkInfo *cip, int *status) { + ChunkData *cdp; + if (cfp->status != CF_READ) + { + *status = CF_INVALID_OP; + return NULL; + } + if (!(cdp = get_chunk_data(cfp->fp, cip))) + { + *status = CF_END_OF_FILE; + return NULL; + } + *status = CF_NORMAL; + return cdp; +} + +void nerv_chunk_file_close(ChunkFile *cfp) { + if (cfp->status != CF_CLOSED) + fclose(cfp->fp); + cfp->status = CF_CLOSED; +} + +void nerv_chunk_file_destroy(ChunkFile *cfp) { + ChunkInfo *i, *ni; + if (cfp->status != CF_CLOSED) fclose(cfp->fp); + for (i = cfp->info; i; i = ni) + { + ni = i->next; + free(i->metadata); + free(i); + } + free(cfp); +} + +void nerv_chunk_data_destroy(ChunkData *cdp) { + fclose(cdp->fp); + free(cdp->data); + free(cdp); +} + +const char *nerv_chunk_file_errstr(int status) { + switch (status) + { + case CF_INVALID_FORMAT: return "invalid format"; + case CF_END_OF_FILE: return "unexpected end of file"; + case CF_SECTION_OVERFLOW: return "section overflow"; + case CF_WRITE_ERROR: return "error while writing"; + case CF_ERR_OPEN_FILE: return "error while opening file"; + case CF_INVALID_OP: return "invalid operation"; + default: return "unknown"; + } + return NULL; +} diff --git a/nerv/lib/io/chunk_file.h b/nerv/lib/io/chunk_file.h new file mode 100644 index 0000000..71f0d03 --- /dev/null +++ b/nerv/lib/io/chunk_file.h @@ -0,0 +1,43 @@ +#ifndef NERV_CHUNK_FILE_H +#define NERV_CHUNK_FILE_H +#include "../../common.h" +enum { + CF_NORMAL, + CF_INVALID_FORMAT, + CF_END_OF_FILE, + CF_SECTION_OVERFLOW, + CF_WRITE_ERROR, + CF_ERR_OPEN_FILE, + CF_INVALID_OP, + CF_READ, + CF_WRITE, + CF_CLOSED +}; + +typedef struct ChunkInfo { + struct ChunkInfo *next; + char *metadata; + off_t offset, length; +} ChunkInfo; + +typedef struct ChunkFile { + FILE *fp; + ChunkInfo *info; + int status; +} ChunkFile; + +typedef struct ChunkData { + FILE *fp; + char *data; +} ChunkData; + +typedef void (*ChunkDataWriter_t)(void *); +ChunkFile *nerv_chunk_file_create(const char *fn, const char *mode, int *status); +int nerv_chunk_file_write_chunkdata(ChunkFile *cfp, const char *mdstr, + ChunkDataWriter_t writer, void *writer_arg); +ChunkData *nerv_chunk_file_get_chunkdata(ChunkFile *cfp, ChunkInfo *cip, int *status); +void nerv_chunk_file_close(ChunkFile *cfp); +void nerv_chunk_file_destroy(ChunkFile *cfp); +void nerv_chunk_data_destroy(ChunkData *cdp); +const char *nerv_chunk_file_errstr(int status); +#endif diff --git a/nerv/luaT/README.md b/nerv/lib/luaT/README.md index 6e9cf0d..6e9cf0d 100644 --- a/nerv/luaT/README.md +++ b/nerv/lib/luaT/README.md diff --git a/nerv/luaT/luaT.c b/nerv/lib/luaT/luaT.c index 7b85ce3..7b85ce3 100644 --- a/nerv/luaT/luaT.c +++ b/nerv/lib/luaT/luaT.c diff --git a/nerv/luaT/luaT.h b/nerv/lib/luaT/luaT.h index 5e8dd2f..5e8dd2f 100644 --- a/nerv/luaT/luaT.h +++ b/nerv/lib/luaT/luaT.h diff --git a/nerv/matrix/generic/mmatrix.c b/nerv/matrix/generic/mmatrix.c index b0f0791..697c9fc 100644 --- a/nerv/matrix/generic/mmatrix.c +++ b/nerv/matrix/generic/mmatrix.c @@ -51,11 +51,11 @@ static void host_matrix_(init)(lua_State *L) { #include "matrix.c" int nerv_matrix_(load)(lua_State *L) { - ChunkData *chunk = luaT_checkudata(L, 1, nerv_chunk_data_tname); + ChunkData *cdp = luaT_checkudata(L, 1, nerv_chunk_data_tname); Matrix *self; int i, j; long nrow, ncol; - FILE *fp = chunk->fp; + FILE *fp = cdp->fp; if (fscanf(fp, "%ld %ld", &nrow, &ncol) != 2) return 0; self = nerv_matrix_(new_)(L, nrow, ncol); @@ -74,12 +74,12 @@ int nerv_matrix_(load)(lua_State *L) { } int nerv_matrix_(save)(lua_State *L) { - ChunkFileHandle *chunk = luaT_checkudata(L, 2, - nerv_chunk_file_handle_tname); + ChunkFile *cfp = luaT_checkudata(L, 2, + nerv_chunk_file_handle_tname); Matrix *self = luaT_checkudata(L, 1, nerv_matrix_(tname)); int i, j; long nrow = self->nrow, ncol = self->ncol; - FILE *fp = chunk->fp; + FILE *fp = cfp->fp; if (fprintf(fp, "%ld %ld\n", nrow, ncol) < 0) return 0; for (i = 0; i < nrow; i++) diff --git a/nerv/nerv-scm-1.rockspec b/nerv/nerv-scm-1.rockspec index d14140a..0b7e4cb 100644 --- a/nerv/nerv-scm-1.rockspec +++ b/nerv/nerv-scm-1.rockspec @@ -1,21 +1,19 @@ package = "nerv" version = "scm-1" source = { - url = "..." -- We don't have one yet + url = "https://github.com/Determinant/nerv.git" } description = { - summary = "An example for the LuaRocks tutorial.", + summary = "A Lua-based toolkit dedicated to high-performance deep neural network learning", detailed = [[ ]], - homepage = "https://github.com/Determinant/nerv", -- We don't have one yet - license = "BSD" -- or whatever you like + homepage = "https://github.com/Determinant/nerv", + license = "BSD" } dependencies = { "lua >= 5.1" - -- If you depend on other rocks, add them here } build = { - -- We'll start here. type = "make", build_variables = { CFLAGS="$(CFLAGS)", |