aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDeterminant <[email protected]>2015-06-23 13:32:42 +0800
committerDeterminant <[email protected]>2015-06-23 13:32:42 +0800
commit8f13607cba9d6cf4fc4a213ba5ae4bcd46f7e18d (patch)
tree28a4c503c2b8ab1e687e0ca8a0cd24a1918401b1
parentb5ca62108775505246f752b9782352b45036ef3c (diff)
separate non-Lua part of io code to a dedicated dir; code clean-up
-rw-r--r--nerv/Makefile8
-rw-r--r--nerv/common.h2
-rw-r--r--nerv/io/chunk_file.c345
-rw-r--r--nerv/io/chunk_file.h19
-rw-r--r--nerv/io/init.lua8
-rw-r--r--nerv/lib/io/chunk_file.c239
-rw-r--r--nerv/lib/io/chunk_file.h43
-rw-r--r--nerv/lib/luaT/README.md (renamed from nerv/luaT/README.md)0
-rw-r--r--nerv/lib/luaT/luaT.c (renamed from nerv/luaT/luaT.c)0
-rw-r--r--nerv/lib/luaT/luaT.h (renamed from nerv/luaT/luaT.h)0
-rw-r--r--nerv/matrix/generic/mmatrix.c10
-rw-r--r--nerv/nerv-scm-1.rockspec10
12 files changed, 385 insertions, 299 deletions
diff --git a/nerv/Makefile b/nerv/Makefile
index b69a63e..4008453 100644
--- a/nerv/Makefile
+++ b/nerv/Makefile
@@ -1,7 +1,7 @@
.PHONY: build install clean
SHELL := /bin/bash
BUILD_DIR := $(CURDIR)/build
-OBJS := nerv.o luaT.o common.o \
+OBJS := nerv.o lib/luaT/luaT.o lib/io/chunk_file.o common.o \
matrix/mmatrix.o matrix/cumatrix.o matrix/init.o matrix/cukernel.o \
io/init.o io/chunk_file.o \
examples/oop_example.o
@@ -19,7 +19,7 @@ INCLUDE += $(CUDA_INCLUDE)
LDFLAGS := -L$(CUDA_BASE)/lib64/ -Wl,-rpath=$(CUDA_BASE)/lib64/ -lcudart -lcublas
CFLAGS := -Wall -Wextra -O2
OBJ_DIR := $(BUILD_DIR)/objs
-SUBDIR := matrix io layer examples nn
+SUBDIR := matrix io layer examples nn lib/io lib/luaT
NVCC := $(CUDA_BASE)/bin/nvcc
NVCC_FLAGS := -Xcompiler -fPIC,-Wall,-Wextra
@@ -39,8 +39,8 @@ $(OBJ_DIR)/matrix/cukernel.o: matrix/cukernel.cu
$(NVCC) -c -o $@ $< $(INCLUDE) $(NVCC_FLAGS)
$(LUA_DIR)/%.lua: %.lua
cp $< $@
-$(OBJ_DIR)/luaT.o:
- gcc -c -o $@ luaT/luaT.c $(INCLUDE) -fPIC
+#$(OBJ_DIR)/luaT.o:
+# gcc -c -o $@ luaT/luaT.c $(INCLUDE) -fPIC
$(LIBS): $(OBJS)
gcc -shared -o $@ $(OBJS) $(LDFLAGS)
diff --git a/nerv/common.h b/nerv/common.h
index e21c7a5..6657dc4 100644
--- a/nerv/common.h
+++ b/nerv/common.h
@@ -3,7 +3,7 @@
#include "lua.h"
#include "lauxlib.h"
#include "lualib.h"
-#include "luaT/luaT.h"
+#include "lib/luaT/luaT.h"
#include <stdio.h>
#include <stdlib.h>
diff --git a/nerv/io/chunk_file.c b/nerv/io/chunk_file.c
index c0b6b9f..e275d9b 100644
--- a/nerv/io/chunk_file.c
+++ b/nerv/io/chunk_file.c
@@ -5,321 +5,136 @@
#include "chunk_file.h"
#define INVALID_FORMAT_ERROR(fn) \
- nerv_error(L, "Invalid chunk file: %s", fn)
+ nerv_error(L, "invalid chunk file: %s", fn)
#define CHECK_FORMAT(exp, ret, fname) \
do { \
if ((exp) != (ret)) INVALID_FORMAT_ERROR(fn); \
} while (0)
-#define CHECK_FILE_OPEN(pfh) \
- do { \
- if ((pfh)->closed) \
- nerv_error(L, "operations on a closed file"); \
- } while (0)
const char *nerv_chunk_file_tname = "nerv.ChunkFile";
const char *nerv_chunk_file_handle_tname = "nerv.ChunkFileHandle";
const char *nerv_chunk_info_tname = "nerv.ChunkInfo";
const char *nerv_chunk_data_tname = "nerv.ChunkData";
-#define PARAM_HEADER_SIZE 16
-
-enum {
- NORMAL,
- INVALID_FORMAT,
- END_OF_FILE,
- SECTION_OVERFLOW,
- WRITE_ERROR
-};
-
-size_t read_chunk_header_plain(FILE *fp, int *status) {
- static char buff[PARAM_HEADER_SIZE];
- int i;
- size_t size = 0;
- *status = NORMAL;
- if (fread(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE)
- {
- if (feof(fp)) *status = END_OF_FILE;
- else *status = INVALID_FORMAT;
- }
- for (i = 0; i < PARAM_HEADER_SIZE; i++)
- if (isdigit(buff[i]))
- size = size * 10 + buff[i] - '0';
-/* fprintf(stderr, "header: %lu\n", size); */
- return size;
-}
-
-#define CHECK_WRITE(status) \
- do { \
- if (status == SECTION_OVERFLOW) \
- nerv_error(L, "section overflowed"); \
- else if (status == WRITE_ERROR) \
- nerv_error(L, "error while writing"); \
- } while (0)
-
-void write_chunk_header_plain(FILE *fp, size_t size, int *status) {
- static char buff[PARAM_HEADER_SIZE];
- int i;
- *status = NORMAL;
- for (i = PARAM_HEADER_SIZE - 3; i > 0; i--, size /= 10)
- buff[i] = size % 10 + '0';
- if (size)
- {
- *status = SECTION_OVERFLOW;
- return;
- }
- buff[0] = '[';
- buff[PARAM_HEADER_SIZE - 2] = ']';
- buff[PARAM_HEADER_SIZE - 1] = '\n';
- if (fwrite(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE)
- {
- *status = WRITE_ERROR;
- return;
- }
-}
-
-ChunkData *get_chunk_data(FILE *fp, ChunkInfo *info) {
- ChunkData *pcd = (ChunkData *)malloc(sizeof(ChunkData));
- pcd->data = (char *)malloc(info->length);
- pcd->fp = fmemopen(pcd->data, info->length, "r");
- assert(fseeko(fp, info->offset, SEEK_SET) == 0);
- if (fread(pcd->data, 1, info->length, fp) != (size_t)info->length)
- return NULL;
- return pcd;
-}
-
-const char *read_chunk_metadata(lua_State *L, FILE *fp, const char *fn) {
-#define LINEBUFF_SIZE 1024
- static char buff[7 + LINEBUFF_SIZE] = "return ";
- CHECK_FORMAT(fgets(buff + 7, LINEBUFF_SIZE, fp), buff + 7, fn);
- /* fprintf(stderr, "metadata: %s\n", buff); */
- return buff;
-}
-
-void write_chunk_metadata(FILE *fp, const char *metadata_str, int *status) {
- size_t size = strlen(metadata_str);
- *status = NORMAL;
- if (fwrite(metadata_str, 1, size, fp) != size ||
- fprintf(fp, "\n") < 0)
+int nerv_lua_chunk_file_new(lua_State *L) {
+ int status;
+ const char *fn = luaL_checkstring(L, 1);
+ ChunkFile *cfp = nerv_chunk_file_create(fn,
+ luaL_checkstring(L, 2),
+ &status);
+ if (status != CF_NORMAL)
{
- *status = WRITE_ERROR;
- return;
+ nerv_error(L, "%s: %s", fn, nerv_chunk_file_errstr(status));
}
- /* fprintf(stderr, "metadata: %s\n", metadata_str); */
-}
-
-
-int nerv_chunk_file_open_write(lua_State *L, const char *fn) {
- FILE *fp = fopen(fn, "w");
- ChunkFileHandle *lfp;
- if (!fp) nerv_error(L, "Error while opening chunk file: %s", fn);
- lfp = (ChunkFileHandle *)malloc(sizeof(ChunkFileHandle));
- lfp->fp = fp;
- lfp->closed = 0;
- luaT_pushudata(L, lfp, nerv_chunk_file_handle_tname);
- lua_setfield(L, -2, "handle");
- luaT_pushmetatable(L, nerv_chunk_file_tname);
- lua_setmetatable(L, -2);
- return 1;
-}
-
-int nerv_chunk_file_open_read(lua_State *L, const char *fn) {
- FILE *fp = fopen(fn, "r");
- int i, status;
- size_t chunk_len;
- off_t offset;
- ChunkFileHandle *lfp;
-
- if (!fp) nerv_error(L, "Error while opening chunk file: %s", fn);
- offset = ftello(fp);
lua_newtable(L);
- /* fprintf(stderr, "%d\n", (int)offset); */
- for (i = 0;; offset += chunk_len, i++)
+ luaT_pushudata(L, cfp, nerv_chunk_file_handle_tname);
+ lua_setfield(L, -2, "handle");
+ if (cfp->status == CF_READ)
{
- ChunkInfo *pci;
- /* fprintf(stderr, "reading chunk %d from %d\n", i, (int)offset); */
- /* skip to the begining of chunk i */
- CHECK_FORMAT(fseeko(fp, offset, SEEK_SET), 0, fn);
- /* read header */
- chunk_len = read_chunk_header_plain(fp, &status);
- if (status == END_OF_FILE) break;
- else if (status == INVALID_FORMAT)
- INVALID_FORMAT_ERROR(fn);
- /* read metadata */
- luaL_loadstring(L, read_chunk_metadata(L, fp, fn));
- CHECK_FORMAT(lua_pcall(L, 0, 1, 0), 0, fn);
- CHECK_FORMAT(lua_istable(L, -1), 1, fn);
- /* stack: obj_table, metadata */
- /* chunk info */
- pci = (ChunkInfo *)malloc(sizeof(ChunkInfo));
- pci->offset = ftello(fp);
- pci->length = chunk_len - (pci->offset - offset);
- /* fprintf(stderr, "%d + %d (skip %lu)\n", (int)pci->offset,
- (int)pci->length, chunk_len); */
- luaT_pushudata(L, pci, nerv_chunk_info_tname);
- lua_setfield(L, -2, "chunk");
- /* stack: obj_table, metadata */
- /* get id */
- lua_getfield(L, -1, "id");
- /* stack: obj_table, metadata, id */
- if (!lua_isstring(L, -1))
- nerv_error(L, "id field in metadata must be a string");
- lua_pushvalue(L, -1);
- /* stack: obj_table, metadata, id, id */
- lua_gettable(L, -4);
- /* stack: obj_table, metadata, id, obj[id] */
- if (!lua_isnil(L, -1))
- nerv_error(L, "conflicting id");
- lua_pop(L, 1);
- /* stack: obj_table, metadata, id */
- lua_pushvalue(L, -2);
- /* stack: obj_table, metadata, id, metadata */
- lua_settable(L, -4);
- /* stack: obj_table, metadata */
- lua_pop(L, 1);
+ ChunkInfo *cip;
+ /* build a table with interpreted metadata in Lua because C API only
+ * provides with linked list with uninterpreted metadata */
+ lua_newtable(L);
+ /* stack: self, metadata_table */
+ for (cip = cfp->info; cip; cip = cip->next)
+ {
+ luaL_loadstring(L, cip->metadata);
+ CHECK_FORMAT(lua_pcall(L, 0, 1, 0), 0, fn);
+ CHECK_FORMAT(lua_istable(L, -1), 1, fn);
+ lua_getfield(L, -1, "id");
+ if (!lua_isstring(L, -1))
+ nerv_error(L, "id field in metadata must be a string");
+ /* stack: ... metadata_table, metadata, id */
+ lua_pushvalue(L, -1);
+ /* stack: ... metadata_table, metadata, id, id */
+ lua_gettable(L, -4);
+ /* stack: ... metadata_table, metadata, id, metadata_table[id] */
+ if (!lua_isnil(L, -1))
+ nerv_error(L, "conflicting id");
+ lua_pop(L, 1);
+ /* stack: ... metadata_table, metadata, id */
+ lua_pushvalue(L, -2);
+ /* stack: ... metadata_table, metadata, id, metadata */
+ lua_settable(L, -4);
+ /* stack: ... metadata_table, metadata */
+ luaT_pushudata(L, cip, nerv_chunk_info_tname);
+ /* stack: ... metadata_table, cip */
+ lua_setfield(L, -2, "_chunk_info");
+ /* stack: ... metadata_table */
+ lua_pop(L, 1);
+ /* stack: ... metadata_table */
+ }
+ lua_setfield(L, -2, "metadata");
+ /* stack: ... */
}
- lua_setfield(L, -2, "metadata");
- lfp = (ChunkFileHandle *)malloc(sizeof(ChunkFileHandle));
- lfp->fp = fp;
- lfp->closed = 0;
- luaT_pushudata(L, lfp, nerv_chunk_file_handle_tname);
- lua_setfield(L, -2, "handle");
luaT_pushmetatable(L, nerv_chunk_file_tname);
lua_setmetatable(L, -2);
return 1;
}
-int nerv_chunk_file_new_(lua_State *L, const char *fn, const char *mode) {
- int rd = 1, bin = 0;
- size_t i, len = strlen(mode);
- for (i = 0; i < len; i++)
- switch (mode[i])
- {
- case 'r': rd = 1; break;
- case 'w': rd = 0; break;
- case 'b': bin = 1; break;
- }
- return rd ? nerv_chunk_file_open_read(L, fn) : \
- nerv_chunk_file_open_write(L, fn);
+static void writer(void *L) {
+ lua_call((lua_State *)L, 2, 0); /* let the write() to write */
}
-int nerv_chunk_file___init(lua_State *L) {
- lua_pushvalue(L, 1);
- return nerv_chunk_file_new_(L, luaL_checkstring(L, 2),
- luaL_checkstring(L, 3));
-}
-
-int nerv_chunk_file_new(lua_State *L) {
- lua_newtable(L);
- return nerv_chunk_file_new_(L, luaL_checkstring(L, 1),
- luaL_checkstring(L, 2));
-}
-
-int nerv_chunk_file_write_chunkdata(lua_State *L) {
- ChunkFileHandle *pfh;
+int nerv_lua_chunk_file_write_chunkdata(lua_State *L) {
int status;
- off_t start;
- size_t size;
- const char *metadata_str = lua_tolstring(L, 2, NULL);
- lua_getfield(L, 1, "handle");
- pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
- CHECK_FILE_OPEN(pfh);
- start = ftello(pfh->fp);
- write_chunk_header_plain(pfh->fp, 0, &status); /* fill zeros */
- CHECK_WRITE(status);
- write_chunk_metadata(pfh->fp, metadata_str, &status);
- CHECK_WRITE(status);
- lua_pushvalue(L, 3);
- lua_getfield(L, -1, "write");
+ ChunkFile *cfp = luaT_checkudata(L, 1, nerv_chunk_file_handle_tname);
+ const char *mdstr = lua_tolstring(L, 2, NULL);
+ lua_getfield(L, 3, "write");
if (!lua_isfunction(L, -1))
nerv_error(L, "\"write\" method must be implemented");
- lua_pushvalue(L, -2);
- lua_pushvalue(L, 4); /* pass handle as parameter to write() */
- lua_call(L, 2, 0); /* let the write() to write */
- lua_pop(L, 1);
- size = ftello(pfh->fp) - start;
- fseeko(pfh->fp, start, SEEK_SET);
- /* write the calced size */
- write_chunk_header_plain(pfh->fp, size, &status);
- CHECK_WRITE(status);
- fseeko(pfh->fp, 0, SEEK_END);
+ lua_pushvalue(L, 3); /* lua writer itself */
+ lua_pushvalue(L, 1); /* pass handle as parameter to write() */
+ nerv_chunk_file_write_chunkdata(cfp, mdstr, writer, (void *)L);
return 0;
}
-int nerv_chunk_file_get_chunkdata(lua_State *L) {
- ChunkFileHandle *pfh;
- ChunkInfo *pci;
- ChunkData *pcd;
- const char *id = luaL_checkstring(L, 2);
-
- lua_getfield(L, 1, "handle");
- pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
- CHECK_FILE_OPEN(pfh);
- lua_pop(L, 1); /* pop handle */
- lua_getfield(L, 1, "metadata");
- /* now stack: self, k, metadata */
- lua_getfield(L, -1, id);
- /* now stack: self, k, metadata, kth{} */
- if (lua_isnil(L, -1)) /* no chunck with the id */
- return 0;
- lua_getfield(L, -1, "chunk");
- pci = luaT_checkudata(L, -1, nerv_chunk_info_tname);
- if (!(pcd = get_chunk_data(pfh->fp, pci)))
- nerv_error(L, "unexpected end of file");
- luaT_pushudata(L, pcd, nerv_chunk_data_tname);
+int nerv_lua_chunk_file_get_chunkdata(lua_State *L) {
+ int status;
+ ChunkFile *cfp = luaT_checkudata(L, 1, nerv_chunk_file_handle_tname);
+ ChunkInfo *cip = luaT_checkudata(L, 2, nerv_chunk_info_tname);
+ ChunkData *cdp = nerv_chunk_file_get_chunkdata(cfp, cip, &status);
+ if (status != CF_NORMAL)
+ nerv_error(L, "%s", nerv_chunk_file_errstr(status));
+ luaT_pushudata(L, cdp, nerv_chunk_data_tname);
return 1;
}
-int nerv_chunk_file_close(lua_State *L) {
- ChunkFileHandle *pfh;
- lua_getfield(L, 1, "handle");
- pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
- CHECK_FILE_OPEN(pfh);
- fclose(pfh->fp);
- pfh->closed = 1;
- return 0;
-}
-
-int nerv_chunk_file_handle_destroy(lua_State *L) {
- ChunkFileHandle *pfh = luaT_checkudata(L, 1,
- nerv_chunk_file_handle_tname);
- if (!pfh->closed) fclose(pfh->fp);
- free(pfh);
+int nerv_lua_chunk_file_close(lua_State *L) {
+ ChunkFile *cfp = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
+ nerv_chunk_file_close(cfp);
return 0;
}
-static int nerv_chunk_info_destroy(lua_State *L) {
- ChunkInfo *pci = luaT_checkudata(L, 1, nerv_chunk_info_tname);
- free(pci);
+int nerv_lua_chunk_file_destroy(lua_State *L) {
+ ChunkFile *cfp = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
+ nerv_chunk_file_destroy(cfp);
return 0;
}
-static int nerv_chunk_data_destroy(lua_State *L) {
- ChunkData *pcd = luaT_checkudata(L, 1, nerv_chunk_data_tname);
- fclose(pcd->fp);
- free(pcd->data);
- free(pcd);
+static int nerv_lua_chunk_data_destroy(lua_State *L) {
+ ChunkData *cdp = luaT_checkudata(L, 1, nerv_chunk_data_tname);
+ nerv_chunk_data_destroy(cdp);
return 0;
}
static const luaL_Reg nerv_chunk_file_methods[] = {
- {"get_chunkdata", nerv_chunk_file_get_chunkdata},
- {"_write_chunkdata", nerv_chunk_file_write_chunkdata},
- {"close", nerv_chunk_file_close},
- {"__init", nerv_chunk_file___init},
+ {"_get_chunkdata", nerv_lua_chunk_file_get_chunkdata},
+ {"_write_chunkdata", nerv_lua_chunk_file_write_chunkdata},
+ {"_close", nerv_lua_chunk_file_close},
{NULL, NULL}
};
void nerv_chunk_file_init(lua_State *L) {
luaT_newmetatable(L, nerv_chunk_file_tname, NULL,
- nerv_chunk_file_new,
+ nerv_lua_chunk_file_new,
NULL, NULL);
luaL_register(L, NULL, nerv_chunk_file_methods);
lua_pop(L, 1);
luaT_newmetatable(L, nerv_chunk_file_handle_tname, NULL,
- NULL, nerv_chunk_file_handle_destroy, NULL);
+ NULL, nerv_lua_chunk_file_destroy, NULL);
luaT_newmetatable(L, nerv_chunk_info_tname, NULL,
- NULL, nerv_chunk_info_destroy, NULL);
+ NULL, NULL, NULL);
luaT_newmetatable(L, nerv_chunk_data_tname, NULL,
- NULL, nerv_chunk_data_destroy, NULL);
+ NULL, nerv_lua_chunk_data_destroy, NULL);
}
-
diff --git a/nerv/io/chunk_file.h b/nerv/io/chunk_file.h
index 9bae59d..71094a5 100644
--- a/nerv/io/chunk_file.h
+++ b/nerv/io/chunk_file.h
@@ -1,23 +1,10 @@
-#ifndef NERV_LAYER_FILE_H
-#define NERV_LAYER_FILE_H
+#ifndef NERV_LUA_CHUNK_FILE_H
+#define NERV_LUA_CHUNK_FILE_H
+#include "../lib/io/chunk_file.h"
extern const char *nerv_chunk_file_tname;
extern const char *nerv_chunk_file_handle_tname;
extern const char *nerv_chunk_info_tname;
extern const char *nerv_chunk_data_tname;
-typedef struct ChunkFileHandle {
- FILE *fp;
- int closed;
-} ChunkFileHandle;
-
-typedef struct ChunkInfo {
- off_t offset, length;
-} ChunkInfo;
-
-typedef struct ChunkData {
- FILE *fp;
- char *data;
-} ChunkData;
-
#endif
diff --git a/nerv/io/init.lua b/nerv/io/init.lua
index 647ff93..eb2e3e5 100644
--- a/nerv/io/init.lua
+++ b/nerv/io/init.lua
@@ -3,7 +3,7 @@ function nerv.ChunkFile:write_chunkdata(metadata, writer)
nerv.error("metadata should be a Lua table")
return
end
- return self:_write_chunkdata(table.tostring(metadata), writer)
+ return self._write_chunkdata(self.handle, table.tostring(metadata), writer)
end
function nerv.ChunkFile:write_chunk(chunk)
@@ -28,10 +28,14 @@ function nerv.ChunkFile:read_chunk(id, global_conf)
local chunk_type = nerv.get_type(metadata.type)
local chunk = chunk_type(id, global_conf)
chunk:set_info(metadata.info)
- chunk:read(self:get_chunkdata(id))
+ chunk:read(self._get_chunkdata(self.handle, metadata._chunk_info))
return chunk
end
+function nerv.ChunkFile:close()
+ self._close(self.handle)
+end
+
local DataReader = nerv.class("nerv.DataReader")
function DataReader:__init(global_conf, reader_conf)
diff --git a/nerv/lib/io/chunk_file.c b/nerv/lib/io/chunk_file.c
new file mode 100644
index 0000000..a305962
--- /dev/null
+++ b/nerv/lib/io/chunk_file.c
@@ -0,0 +1,239 @@
+#include "../../common.h"
+#include "chunk_file.h"
+#include <stdlib.h>
+#include <string.h>
+#define PARAM_HEADER_SIZE 16
+
+static size_t read_chunk_header_plain(FILE *fp, int *status) {
+ static char buff[PARAM_HEADER_SIZE];
+ int i;
+ size_t size = 0;
+ if (fread(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE)
+ {
+ if (feof(fp)) *status = CF_END_OF_FILE;
+ else
+ {
+ *status = CF_INVALID_FORMAT;
+ return 0;
+ }
+ }
+ else *status = CF_NORMAL;
+ for (i = 0; i < PARAM_HEADER_SIZE; i++)
+ if (isdigit(buff[i]))
+ size = size * 10 + buff[i] - '0';
+/* fprintf(stderr, "header: %lu\n", size); */
+ return size;
+}
+
+static void write_chunk_header_plain(FILE *fp, size_t size, int *status) {
+ static char buff[PARAM_HEADER_SIZE];
+ int i;
+ for (i = PARAM_HEADER_SIZE - 3; i > 0; i--, size /= 10)
+ buff[i] = size % 10 + '0';
+ if (size)
+ {
+ *status = CF_SECTION_OVERFLOW;
+ return;
+ }
+ buff[0] = '[';
+ buff[PARAM_HEADER_SIZE - 2] = ']';
+ buff[PARAM_HEADER_SIZE - 1] = '\n';
+ if (fwrite(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE)
+ {
+ *status = CF_WRITE_ERROR;
+ return;
+ }
+ *status = CF_NORMAL;
+}
+
+static ChunkData *get_chunk_data(FILE *fp, ChunkInfo *info) {
+ ChunkData *cdp = (ChunkData *)malloc(sizeof(ChunkData));
+ cdp->data = (char *)malloc(info->length);
+ cdp->fp = fmemopen(cdp->data, info->length, "r");
+ assert(fseeko(fp, info->offset, SEEK_SET) == 0);
+ if (fread(cdp->data, 1, info->length, fp) != (size_t)info->length)
+ return NULL;
+ return cdp;
+}
+
+static const char *read_chunk_metadata(FILE *fp, const char *fn, int *status) {
+#define LINEBUFF_SIZE 1024
+#define LUA_RETURN "return "
+#define LUA_RETURN_LEN (sizeof(LUA_RETURN) - 1)
+ static char buff[LUA_RETURN_LEN + LINEBUFF_SIZE] = LUA_RETURN;
+ *status = fgets(buff + LUA_RETURN_LEN,
+ LINEBUFF_SIZE, fp) == (buff + LUA_RETURN_LEN) ? \
+ CF_NORMAL : CF_INVALID_FORMAT;
+ fprintf(stderr, "metadata: %s\n", buff);
+ return buff;
+}
+
+static void write_chunk_metadata(FILE *fp, const char *mdstr, int *status) {
+ size_t size = strlen(mdstr);
+ if (fwrite(mdstr, 1, size, fp) != size ||
+ fprintf(fp, "\n") < 0)
+ {
+ *status = CF_WRITE_ERROR;
+ return;
+ }
+ /* fprintf(stderr, "metadata: %s\n", metadata_str); */
+ *status = CF_NORMAL;
+}
+
+static ChunkFile *open_write(const char *fn, int *status) {
+ ChunkFile *cfp;
+ FILE *fp = fopen(fn, "w");
+
+ if (!fp)
+ {
+ *status = CF_ERR_OPEN_FILE;
+ return NULL;
+ }
+ cfp = (ChunkFile *)malloc(sizeof(ChunkFile));
+ cfp->fp = fp;
+ cfp->status = CF_WRITE;
+ *status = CF_NORMAL;
+ return cfp;
+}
+
+static ChunkFile *open_read(const char *fn, int *status) {
+ size_t chunk_len;
+ off_t offset;
+ int i;
+ const char *mdstr;
+ ChunkFile *cfp;
+ ChunkInfo *head = NULL;
+ FILE *fp = fopen(fn, "r");
+
+ if (!fp)
+ {
+ *status = CF_ERR_OPEN_FILE;
+ return NULL;
+ }
+ cfp = (ChunkFile *)malloc(sizeof(ChunkFile));
+ cfp->fp = fp;
+ cfp->status = CF_READ;
+ offset = ftello(fp);
+ /* fprintf(stderr, "%d\n", (int)offset); */
+ for (i = 0;; offset += chunk_len, i++)
+ {
+ ChunkInfo *cip;
+ fprintf(stderr, "reading chunk %d from %d\n", i, (int)offset);
+ /* skip to the begining of chunk i */
+ if (fseeko(fp, offset, SEEK_SET) != 0)
+ {
+ *status = CF_INVALID_FORMAT;
+ return NULL;
+ }
+ /* read header */
+ chunk_len = read_chunk_header_plain(fp, status);
+ if (*status == CF_END_OF_FILE) break;
+ if (*status != CF_NORMAL)
+ return NULL;
+ cip = (ChunkInfo *)malloc(sizeof(ChunkInfo));
+ /* read metadata */
+ mdstr = read_chunk_metadata(fp, fn, status);
+ if (*status != CF_NORMAL)
+ return NULL;
+ cip->metadata = strdup(mdstr);
+ cip->offset = ftello(fp);
+ cip->length = chunk_len - (cip->offset - offset);
+ /* fprintf(stderr, "%d + %d (skip %lu)\n", (int)cip->offset,
+ (int)cip->length, chunk_len); */
+ cip->next = head;
+ head = cip;
+ }
+ *status = CF_NORMAL;
+ cfp->info = head;
+ return cfp;
+}
+
+ChunkFile *nerv_chunk_file_create(const char *fn, const char *mode, int *status) {
+ int rd = 1, bin = 0;
+ size_t i, len = strlen(mode);
+ for (i = 0; i < len; i++)
+ switch (mode[i])
+ {
+ case 'r': rd = 1; break;
+ case 'w': rd = 0; break;
+ case 'b': bin = 1; break;
+ }
+ return rd ? open_read(fn, status) : \
+ open_write(fn, status);
+}
+
+int nerv_chunk_file_write_chunkdata(ChunkFile *cfp, const char *mdstr,
+ ChunkDataWriter_t writer, void *writer_arg) {
+ int status;
+ off_t start;
+ size_t size;
+ if (cfp->status != CF_WRITE)
+ return CF_INVALID_OP;
+ start = ftello(cfp->fp);
+ write_chunk_header_plain(cfp->fp, 0, &status); /* fill zeros */
+ if (status != CF_NORMAL) return status;
+ write_chunk_metadata(cfp->fp, mdstr, &status);
+ if (status != CF_NORMAL) return status;
+ writer(writer_arg);
+ size = ftello(cfp->fp) - start;
+ fseeko(cfp->fp, start, SEEK_SET);
+ /* write the calced size */
+ write_chunk_header_plain(cfp->fp, size, &status);
+ if (status != CF_NORMAL) return status;
+ fseeko(cfp->fp, 0, SEEK_END);
+ return CF_NORMAL;
+}
+
+ChunkData *nerv_chunk_file_get_chunkdata(ChunkFile *cfp, ChunkInfo *cip, int *status) {
+ ChunkData *cdp;
+ if (cfp->status != CF_READ)
+ {
+ *status = CF_INVALID_OP;
+ return NULL;
+ }
+ if (!(cdp = get_chunk_data(cfp->fp, cip)))
+ {
+ *status = CF_END_OF_FILE;
+ return NULL;
+ }
+ *status = CF_NORMAL;
+ return cdp;
+}
+
+void nerv_chunk_file_close(ChunkFile *cfp) {
+ if (cfp->status != CF_CLOSED)
+ fclose(cfp->fp);
+ cfp->status = CF_CLOSED;
+}
+
+void nerv_chunk_file_destroy(ChunkFile *cfp) {
+ ChunkInfo *i, *ni;
+ if (cfp->status != CF_CLOSED) fclose(cfp->fp);
+ for (i = cfp->info; i; i = ni)
+ {
+ ni = i->next;
+ free(i->metadata);
+ free(i);
+ }
+ free(cfp);
+}
+
+void nerv_chunk_data_destroy(ChunkData *cdp) {
+ fclose(cdp->fp);
+ free(cdp->data);
+ free(cdp);
+}
+
+const char *nerv_chunk_file_errstr(int status) {
+ switch (status)
+ {
+ case CF_INVALID_FORMAT: return "invalid format";
+ case CF_END_OF_FILE: return "unexpected end of file";
+ case CF_SECTION_OVERFLOW: return "section overflow";
+ case CF_WRITE_ERROR: return "error while writing";
+ case CF_ERR_OPEN_FILE: return "error while opening file";
+ case CF_INVALID_OP: return "invalid operation";
+ default: return "unknown";
+ }
+ return NULL;
+}
diff --git a/nerv/lib/io/chunk_file.h b/nerv/lib/io/chunk_file.h
new file mode 100644
index 0000000..71f0d03
--- /dev/null
+++ b/nerv/lib/io/chunk_file.h
@@ -0,0 +1,43 @@
+#ifndef NERV_CHUNK_FILE_H
+#define NERV_CHUNK_FILE_H
+#include "../../common.h"
+enum {
+ CF_NORMAL,
+ CF_INVALID_FORMAT,
+ CF_END_OF_FILE,
+ CF_SECTION_OVERFLOW,
+ CF_WRITE_ERROR,
+ CF_ERR_OPEN_FILE,
+ CF_INVALID_OP,
+ CF_READ,
+ CF_WRITE,
+ CF_CLOSED
+};
+
+typedef struct ChunkInfo {
+ struct ChunkInfo *next;
+ char *metadata;
+ off_t offset, length;
+} ChunkInfo;
+
+typedef struct ChunkFile {
+ FILE *fp;
+ ChunkInfo *info;
+ int status;
+} ChunkFile;
+
+typedef struct ChunkData {
+ FILE *fp;
+ char *data;
+} ChunkData;
+
+typedef void (*ChunkDataWriter_t)(void *);
+ChunkFile *nerv_chunk_file_create(const char *fn, const char *mode, int *status);
+int nerv_chunk_file_write_chunkdata(ChunkFile *cfp, const char *mdstr,
+ ChunkDataWriter_t writer, void *writer_arg);
+ChunkData *nerv_chunk_file_get_chunkdata(ChunkFile *cfp, ChunkInfo *cip, int *status);
+void nerv_chunk_file_close(ChunkFile *cfp);
+void nerv_chunk_file_destroy(ChunkFile *cfp);
+void nerv_chunk_data_destroy(ChunkData *cdp);
+const char *nerv_chunk_file_errstr(int status);
+#endif
diff --git a/nerv/luaT/README.md b/nerv/lib/luaT/README.md
index 6e9cf0d..6e9cf0d 100644
--- a/nerv/luaT/README.md
+++ b/nerv/lib/luaT/README.md
diff --git a/nerv/luaT/luaT.c b/nerv/lib/luaT/luaT.c
index 7b85ce3..7b85ce3 100644
--- a/nerv/luaT/luaT.c
+++ b/nerv/lib/luaT/luaT.c
diff --git a/nerv/luaT/luaT.h b/nerv/lib/luaT/luaT.h
index 5e8dd2f..5e8dd2f 100644
--- a/nerv/luaT/luaT.h
+++ b/nerv/lib/luaT/luaT.h
diff --git a/nerv/matrix/generic/mmatrix.c b/nerv/matrix/generic/mmatrix.c
index b0f0791..697c9fc 100644
--- a/nerv/matrix/generic/mmatrix.c
+++ b/nerv/matrix/generic/mmatrix.c
@@ -51,11 +51,11 @@ static void host_matrix_(init)(lua_State *L) {
#include "matrix.c"
int nerv_matrix_(load)(lua_State *L) {
- ChunkData *chunk = luaT_checkudata(L, 1, nerv_chunk_data_tname);
+ ChunkData *cdp = luaT_checkudata(L, 1, nerv_chunk_data_tname);
Matrix *self;
int i, j;
long nrow, ncol;
- FILE *fp = chunk->fp;
+ FILE *fp = cdp->fp;
if (fscanf(fp, "%ld %ld", &nrow, &ncol) != 2)
return 0;
self = nerv_matrix_(new_)(L, nrow, ncol);
@@ -74,12 +74,12 @@ int nerv_matrix_(load)(lua_State *L) {
}
int nerv_matrix_(save)(lua_State *L) {
- ChunkFileHandle *chunk = luaT_checkudata(L, 2,
- nerv_chunk_file_handle_tname);
+ ChunkFile *cfp = luaT_checkudata(L, 2,
+ nerv_chunk_file_handle_tname);
Matrix *self = luaT_checkudata(L, 1, nerv_matrix_(tname));
int i, j;
long nrow = self->nrow, ncol = self->ncol;
- FILE *fp = chunk->fp;
+ FILE *fp = cfp->fp;
if (fprintf(fp, "%ld %ld\n", nrow, ncol) < 0)
return 0;
for (i = 0; i < nrow; i++)
diff --git a/nerv/nerv-scm-1.rockspec b/nerv/nerv-scm-1.rockspec
index d14140a..0b7e4cb 100644
--- a/nerv/nerv-scm-1.rockspec
+++ b/nerv/nerv-scm-1.rockspec
@@ -1,21 +1,19 @@
package = "nerv"
version = "scm-1"
source = {
- url = "..." -- We don't have one yet
+ url = "https://github.com/Determinant/nerv.git"
}
description = {
- summary = "An example for the LuaRocks tutorial.",
+ summary = "A Lua-based toolkit dedicated to high-performance deep neural network learning",
detailed = [[
]],
- homepage = "https://github.com/Determinant/nerv", -- We don't have one yet
- license = "BSD" -- or whatever you like
+ homepage = "https://github.com/Determinant/nerv",
+ license = "BSD"
}
dependencies = {
"lua >= 5.1"
- -- If you depend on other rocks, add them here
}
build = {
- -- We'll start here.
type = "make",
build_variables = {
CFLAGS="$(CFLAGS)",