separate non-Lua part of io code to a dedicated dir; code clean-up

12 files changed, 385 insertions, 299 deletions

diff --git a/nerv/Makefile b/nerv/Makefile
index b69a63e..4008453 100644
--- a/nerv/Makefile
+++ b/nerv/Makefile
@@ -1,7 +1,7 @@
 .PHONY: build install clean
 SHELL := /bin/bash
 BUILD_DIR := $(CURDIR)/build
-OBJS := nerv.o luaT.o common.o \
+OBJS := nerv.o lib/luaT/luaT.o lib/io/chunk_file.o common.o \
 		matrix/mmatrix.o matrix/cumatrix.o matrix/init.o matrix/cukernel.o \
 		io/init.o io/chunk_file.o \
 		examples/oop_example.o
@@ -19,7 +19,7 @@ INCLUDE += $(CUDA_INCLUDE)
 LDFLAGS := -L$(CUDA_BASE)/lib64/  -Wl,-rpath=$(CUDA_BASE)/lib64/ -lcudart -lcublas
 CFLAGS := -Wall -Wextra -O2
 OBJ_DIR := $(BUILD_DIR)/objs
-SUBDIR := matrix io layer examples nn
+SUBDIR := matrix io layer examples nn lib/io lib/luaT
 NVCC := $(CUDA_BASE)/bin/nvcc
 NVCC_FLAGS := -Xcompiler -fPIC,-Wall,-Wextra
 
@@ -39,8 +39,8 @@ $(OBJ_DIR)/matrix/cukernel.o: matrix/cukernel.cu
 	$(NVCC) -c -o $@ $< $(INCLUDE) $(NVCC_FLAGS)
 $(LUA_DIR)/%.lua: %.lua
 	cp $< $@
-$(OBJ_DIR)/luaT.o:
-	gcc -c -o $@ luaT/luaT.c $(INCLUDE) -fPIC
+#$(OBJ_DIR)/luaT.o:
+#	gcc -c -o $@ luaT/luaT.c $(INCLUDE) -fPIC
 $(LIBS): $(OBJS)
 	gcc -shared -o $@ $(OBJS) $(LDFLAGS)
 
diff --git a/nerv/common.h b/nerv/common.h
index e21c7a5..6657dc4 100644
--- a/nerv/common.h
+++ b/nerv/common.h
@@ -3,7 +3,7 @@
 #include "lua.h"
 #include "lauxlib.h"
 #include "lualib.h"
-#include "luaT/luaT.h"
+#include "lib/luaT/luaT.h"
 #include <stdio.h>
 #include <stdlib.h>
 
diff --git a/nerv/io/chunk_file.c b/nerv/io/chunk_file.c
index c0b6b9f..e275d9b 100644
--- a/nerv/io/chunk_file.c
+++ b/nerv/io/chunk_file.c
@@ -5,321 +5,136 @@
 #include "chunk_file.h"
 
 #define INVALID_FORMAT_ERROR(fn) \
-    nerv_error(L, "Invalid chunk file: %s", fn)
+    nerv_error(L, "invalid chunk file: %s", fn)
 #define CHECK_FORMAT(exp, ret, fname) \
     do { \
         if ((exp) != (ret)) INVALID_FORMAT_ERROR(fn); \
     } while (0)
-#define CHECK_FILE_OPEN(pfh) \
-    do { \
-        if ((pfh)->closed) \
-            nerv_error(L, "operations on a closed file"); \
-    } while (0)
 
 const char *nerv_chunk_file_tname = "nerv.ChunkFile";
 const char *nerv_chunk_file_handle_tname = "nerv.ChunkFileHandle";
 const char *nerv_chunk_info_tname = "nerv.ChunkInfo";
 const char *nerv_chunk_data_tname = "nerv.ChunkData";
 
-#define PARAM_HEADER_SIZE 16
-
-enum {
-    NORMAL,
-    INVALID_FORMAT,
-    END_OF_FILE,
-    SECTION_OVERFLOW,
-    WRITE_ERROR
-};
-
-size_t read_chunk_header_plain(FILE *fp, int *status) {
-    static char buff[PARAM_HEADER_SIZE];
-    int i;
-    size_t size = 0;
-    *status = NORMAL;
-    if (fread(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE)
-    {
-        if (feof(fp)) *status = END_OF_FILE;
-        else *status = INVALID_FORMAT;
-    }
-    for (i = 0; i < PARAM_HEADER_SIZE; i++)
-        if (isdigit(buff[i]))
-            size = size * 10 + buff[i] - '0';
-/*    fprintf(stderr, "header: %lu\n", size); */
-    return size;
-}
-
-#define CHECK_WRITE(status) \
-    do { \
-        if (status == SECTION_OVERFLOW) \
-            nerv_error(L, "section overflowed"); \
-        else if (status == WRITE_ERROR) \
-            nerv_error(L, "error while writing"); \
-    } while (0)
-
-void write_chunk_header_plain(FILE *fp, size_t size, int *status) {
-    static char buff[PARAM_HEADER_SIZE];
-    int i;
-    *status = NORMAL;
-    for (i = PARAM_HEADER_SIZE - 3; i > 0; i--, size /= 10)
-        buff[i] = size % 10 + '0';
-    if (size)
-    {
-        *status = SECTION_OVERFLOW;
-        return;
-    }
-    buff[0] = '[';
-    buff[PARAM_HEADER_SIZE - 2] = ']';
-    buff[PARAM_HEADER_SIZE - 1] = '\n';
-    if (fwrite(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE)
-    {
-        *status = WRITE_ERROR;
-        return;
-    }
-}
-
-ChunkData *get_chunk_data(FILE *fp, ChunkInfo *info) {
-    ChunkData *pcd = (ChunkData *)malloc(sizeof(ChunkData));
-    pcd->data = (char *)malloc(info->length);
-    pcd->fp = fmemopen(pcd->data, info->length, "r");
-    assert(fseeko(fp, info->offset, SEEK_SET) == 0);
-    if (fread(pcd->data, 1, info->length, fp) != (size_t)info->length)
-        return NULL;
-    return pcd;
-}
-
-const char *read_chunk_metadata(lua_State *L, FILE *fp, const char *fn) {
-#define LINEBUFF_SIZE 1024
-    static char buff[7 + LINEBUFF_SIZE] = "return ";
-    CHECK_FORMAT(fgets(buff + 7, LINEBUFF_SIZE, fp), buff + 7, fn);
-    /* fprintf(stderr, "metadata: %s\n", buff); */
-    return buff;
-}
-
-void write_chunk_metadata(FILE *fp, const char *metadata_str, int *status) {
-    size_t size = strlen(metadata_str);
-    *status = NORMAL;
-    if (fwrite(metadata_str, 1, size, fp) != size ||
-        fprintf(fp, "\n") < 0)
+int nerv_lua_chunk_file_new(lua_State *L) {
+    int status;
+    const char *fn = luaL_checkstring(L, 1);
+    ChunkFile *cfp = nerv_chunk_file_create(fn,
+                                            luaL_checkstring(L, 2),
+                                            &status);
+    if (status != CF_NORMAL)
     {
-        *status = WRITE_ERROR;
-        return;
+        nerv_error(L, "%s: %s", fn, nerv_chunk_file_errstr(status));
     }
-    /* fprintf(stderr, "metadata: %s\n", metadata_str); */
-}
-
-
-int nerv_chunk_file_open_write(lua_State *L, const char *fn) {
-    FILE *fp = fopen(fn, "w");
-    ChunkFileHandle *lfp;
-    if (!fp) nerv_error(L, "Error while opening chunk file: %s", fn);
-    lfp = (ChunkFileHandle *)malloc(sizeof(ChunkFileHandle));
-    lfp->fp = fp;
-    lfp->closed = 0;
-    luaT_pushudata(L, lfp, nerv_chunk_file_handle_tname);
-    lua_setfield(L, -2, "handle");
-    luaT_pushmetatable(L, nerv_chunk_file_tname);
-    lua_setmetatable(L, -2);
-    return 1;
-}
-
-int nerv_chunk_file_open_read(lua_State *L, const char *fn) {
-    FILE *fp = fopen(fn, "r");
-    int i, status;
-    size_t chunk_len;
-    off_t offset;
-    ChunkFileHandle *lfp;
-
-    if (!fp) nerv_error(L, "Error while opening chunk file: %s", fn);
-    offset = ftello(fp);
     lua_newtable(L);
-    /* fprintf(stderr, "%d\n", (int)offset); */
-    for (i = 0;; offset += chunk_len, i++)
+    luaT_pushudata(L, cfp, nerv_chunk_file_handle_tname);
+    lua_setfield(L, -2, "handle");
+    if (cfp->status == CF_READ)
     {
-        ChunkInfo *pci;
-        /* fprintf(stderr, "reading chunk %d from %d\n", i, (int)offset); */
-        /* skip to the begining of chunk i */
-        CHECK_FORMAT(fseeko(fp, offset, SEEK_SET), 0, fn);
-        /* read header */
-        chunk_len = read_chunk_header_plain(fp, &status);
-        if (status == END_OF_FILE) break;
-        else if (status == INVALID_FORMAT)
-            INVALID_FORMAT_ERROR(fn);
-        /* read metadata */
-        luaL_loadstring(L, read_chunk_metadata(L, fp, fn));
-        CHECK_FORMAT(lua_pcall(L, 0, 1, 0), 0, fn);
-        CHECK_FORMAT(lua_istable(L, -1), 1, fn);
-        /* stack: obj_table, metadata */
-        /* chunk info */
-        pci = (ChunkInfo *)malloc(sizeof(ChunkInfo));
-        pci->offset = ftello(fp);
-        pci->length = chunk_len - (pci->offset - offset);
-        /* fprintf(stderr, "%d + %d (skip %lu)\n", (int)pci->offset,
-                (int)pci->length, chunk_len); */
-        luaT_pushudata(L, pci, nerv_chunk_info_tname);
-        lua_setfield(L, -2, "chunk");
-        /* stack: obj_table, metadata */
-        /* get id */
-        lua_getfield(L, -1, "id");
-        /* stack: obj_table, metadata, id */
-        if (!lua_isstring(L, -1))
-            nerv_error(L, "id field in metadata must be a string");
-        lua_pushvalue(L, -1);
-        /* stack: obj_table, metadata, id, id */
-        lua_gettable(L, -4);
-        /* stack: obj_table, metadata, id, obj[id] */
-        if (!lua_isnil(L, -1))
-            nerv_error(L, "conflicting id");
-        lua_pop(L, 1);
-        /* stack: obj_table, metadata, id */
-        lua_pushvalue(L, -2);
-        /* stack: obj_table, metadata, id, metadata */
-        lua_settable(L, -4);
-        /* stack: obj_table, metadata */
-        lua_pop(L, 1);
+        ChunkInfo *cip;
+        /* build a table with interpreted metadata in Lua because C API only
+         * provides with linked list with uninterpreted metadata */
+        lua_newtable(L);
+        /* stack: self, metadata_table */
+        for (cip = cfp->info; cip; cip = cip->next)
+        {
+            luaL_loadstring(L, cip->metadata);
+            CHECK_FORMAT(lua_pcall(L, 0, 1, 0), 0, fn);
+            CHECK_FORMAT(lua_istable(L, -1), 1, fn);
+            lua_getfield(L, -1, "id");
+            if (!lua_isstring(L, -1))
+                nerv_error(L, "id field in metadata must be a string");
+            /* stack: ... metadata_table, metadata, id */
+            lua_pushvalue(L, -1);
+            /* stack: ... metadata_table, metadata, id, id */
+            lua_gettable(L, -4);
+            /* stack: ... metadata_table, metadata, id, metadata_table[id] */
+            if (!lua_isnil(L, -1))
+                nerv_error(L, "conflicting id");
+            lua_pop(L, 1);
+            /* stack: ... metadata_table, metadata, id */
+            lua_pushvalue(L, -2);
+            /* stack: ... metadata_table, metadata, id, metadata */
+            lua_settable(L, -4);
+            /* stack: ... metadata_table, metadata */
+            luaT_pushudata(L, cip, nerv_chunk_info_tname);
+            /* stack: ... metadata_table, cip */
+            lua_setfield(L, -2, "_chunk_info");
+            /* stack: ... metadata_table */
+            lua_pop(L, 1);
+            /* stack: ... metadata_table */
+        }
+        lua_setfield(L, -2, "metadata");
+        /* stack: ... */
     }
-    lua_setfield(L, -2, "metadata");
-    lfp = (ChunkFileHandle *)malloc(sizeof(ChunkFileHandle));
-    lfp->fp = fp;
-    lfp->closed = 0;
-    luaT_pushudata(L, lfp, nerv_chunk_file_handle_tname);
-    lua_setfield(L, -2, "handle");
     luaT_pushmetatable(L, nerv_chunk_file_tname);
     lua_setmetatable(L, -2);
     return 1;
 }
 
-int nerv_chunk_file_new_(lua_State *L, const char *fn, const char *mode) {
-    int rd = 1, bin = 0;
-    size_t i, len = strlen(mode);
-    for (i = 0; i < len; i++)
-        switch (mode[i])
-        {
-            case 'r': rd = 1; break;
-            case 'w': rd = 0; break;
-            case 'b': bin = 1; break;
-        }
-    return rd ? nerv_chunk_file_open_read(L, fn) : \
-                nerv_chunk_file_open_write(L, fn);
+static void writer(void *L) {
+    lua_call((lua_State *)L, 2, 0); /* let the write() to write */
 }
 
-int nerv_chunk_file___init(lua_State *L) {
-    lua_pushvalue(L, 1);
-    return nerv_chunk_file_new_(L, luaL_checkstring(L, 2),
-                                    luaL_checkstring(L, 3));
-}
-
-int nerv_chunk_file_new(lua_State *L) {
-    lua_newtable(L);
-    return nerv_chunk_file_new_(L, luaL_checkstring(L, 1),
-                                    luaL_checkstring(L, 2));
-}
-
-int nerv_chunk_file_write_chunkdata(lua_State *L) {
-    ChunkFileHandle *pfh;
+int nerv_lua_chunk_file_write_chunkdata(lua_State *L) {
     int status;
-    off_t start;
-    size_t size;
-    const char *metadata_str = lua_tolstring(L, 2, NULL);
-    lua_getfield(L, 1, "handle");
-    pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
-    CHECK_FILE_OPEN(pfh);
-    start = ftello(pfh->fp);
-    write_chunk_header_plain(pfh->fp, 0, &status); /* fill zeros */
-    CHECK_WRITE(status);
-    write_chunk_metadata(pfh->fp, metadata_str, &status);
-    CHECK_WRITE(status);
-    lua_pushvalue(L, 3);
-    lua_getfield(L, -1, "write");
+    ChunkFile *cfp = luaT_checkudata(L, 1, nerv_chunk_file_handle_tname);
+    const char *mdstr = lua_tolstring(L, 2, NULL);
+    lua_getfield(L, 3, "write");
     if (!lua_isfunction(L, -1))
         nerv_error(L, "\"write\" method must be implemented");
-    lua_pushvalue(L, -2);
-    lua_pushvalue(L, 4); /* pass handle as parameter to write() */
-    lua_call(L, 2, 0); /* let the write() to write */
-    lua_pop(L, 1);
-    size = ftello(pfh->fp) - start;
-    fseeko(pfh->fp, start, SEEK_SET);
-    /* write the calced size */
-    write_chunk_header_plain(pfh->fp, size, &status);
-    CHECK_WRITE(status);
-    fseeko(pfh->fp, 0, SEEK_END);
+    lua_pushvalue(L, 3); /* lua writer itself */
+    lua_pushvalue(L, 1); /* pass handle as parameter to write() */
+    nerv_chunk_file_write_chunkdata(cfp, mdstr, writer, (void *)L);
     return 0;
 }
 
-int nerv_chunk_file_get_chunkdata(lua_State *L) {
-    ChunkFileHandle *pfh;
-    ChunkInfo *pci;
-    ChunkData *pcd;
-    const char *id = luaL_checkstring(L, 2);
-
-    lua_getfield(L, 1, "handle");
-    pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
-    CHECK_FILE_OPEN(pfh);
-    lua_pop(L, 1); /* pop handle */
-    lua_getfield(L, 1, "metadata");
-    /* now stack: self, k, metadata */
-    lua_getfield(L, -1, id);
-    /* now stack: self, k, metadata, kth{} */
-    if (lua_isnil(L, -1)) /* no chunck with the id */
-        return 0;
-    lua_getfield(L, -1, "chunk");
-    pci = luaT_checkudata(L, -1, nerv_chunk_info_tname);
-    if (!(pcd = get_chunk_data(pfh->fp, pci)))
-        nerv_error(L, "unexpected end of file");
-    luaT_pushudata(L, pcd, nerv_chunk_data_tname);
+int nerv_lua_chunk_file_get_chunkdata(lua_State *L) {
+    int status;
+    ChunkFile *cfp = luaT_checkudata(L, 1, nerv_chunk_file_handle_tname);
+    ChunkInfo *cip = luaT_checkudata(L, 2, nerv_chunk_info_tname);
+    ChunkData *cdp = nerv_chunk_file_get_chunkdata(cfp, cip, &status);
+    if (status != CF_NORMAL)
+        nerv_error(L, "%s", nerv_chunk_file_errstr(status));
+    luaT_pushudata(L, cdp, nerv_chunk_data_tname);
     return 1;
 }
 
-int nerv_chunk_file_close(lua_State *L) {
-    ChunkFileHandle *pfh;
-    lua_getfield(L, 1, "handle");
-    pfh = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
-    CHECK_FILE_OPEN(pfh);
-    fclose(pfh->fp);
-    pfh->closed = 1;
-    return 0;
-}
-
-int nerv_chunk_file_handle_destroy(lua_State *L) {
-    ChunkFileHandle *pfh = luaT_checkudata(L, 1,
-                                nerv_chunk_file_handle_tname);
-    if (!pfh->closed) fclose(pfh->fp);
-    free(pfh);
+int nerv_lua_chunk_file_close(lua_State *L) {
+    ChunkFile *cfp = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
+    nerv_chunk_file_close(cfp);
     return 0;
 }
 
-static int nerv_chunk_info_destroy(lua_State *L) {
-    ChunkInfo *pci = luaT_checkudata(L, 1, nerv_chunk_info_tname);
-    free(pci);
+int nerv_lua_chunk_file_destroy(lua_State *L) {
+    ChunkFile *cfp = luaT_checkudata(L, -1, nerv_chunk_file_handle_tname);
+    nerv_chunk_file_destroy(cfp);
     return 0;
 }
 
-static int nerv_chunk_data_destroy(lua_State *L) {
-    ChunkData *pcd = luaT_checkudata(L, 1, nerv_chunk_data_tname);
-    fclose(pcd->fp);
-    free(pcd->data);
-    free(pcd);
+static int nerv_lua_chunk_data_destroy(lua_State *L) {
+    ChunkData *cdp = luaT_checkudata(L, 1, nerv_chunk_data_tname);
+    nerv_chunk_data_destroy(cdp);
     return 0;
 }
 
 static const luaL_Reg nerv_chunk_file_methods[] = {
-    {"get_chunkdata", nerv_chunk_file_get_chunkdata},
-    {"_write_chunkdata", nerv_chunk_file_write_chunkdata},
-    {"close", nerv_chunk_file_close},
-    {"__init", nerv_chunk_file___init},
+    {"_get_chunkdata", nerv_lua_chunk_file_get_chunkdata},
+    {"_write_chunkdata", nerv_lua_chunk_file_write_chunkdata},
+    {"_close", nerv_lua_chunk_file_close},
     {NULL, NULL}
 };
 
 void nerv_chunk_file_init(lua_State *L) {
     luaT_newmetatable(L, nerv_chunk_file_tname, NULL,
-                        nerv_chunk_file_new,
+                        nerv_lua_chunk_file_new,
                         NULL, NULL);
     luaL_register(L, NULL, nerv_chunk_file_methods);
     lua_pop(L, 1);
     luaT_newmetatable(L, nerv_chunk_file_handle_tname, NULL,
-                        NULL, nerv_chunk_file_handle_destroy, NULL);
+                        NULL, nerv_lua_chunk_file_destroy, NULL);
     luaT_newmetatable(L, nerv_chunk_info_tname, NULL,
-                        NULL, nerv_chunk_info_destroy, NULL);
+                        NULL, NULL, NULL);
     luaT_newmetatable(L, nerv_chunk_data_tname, NULL,
-                        NULL, nerv_chunk_data_destroy, NULL);
+                        NULL, nerv_lua_chunk_data_destroy, NULL);
 }
-
diff --git a/nerv/io/chunk_file.h b/nerv/io/chunk_file.h
index 9bae59d..71094a5 100644
--- a/nerv/io/chunk_file.h
+++ b/nerv/io/chunk_file.h
@@ -1,23 +1,10 @@
-#ifndef NERV_LAYER_FILE_H
-#define NERV_LAYER_FILE_H
+#ifndef NERV_LUA_CHUNK_FILE_H
+#define NERV_LUA_CHUNK_FILE_H
+#include "../lib/io/chunk_file.h"
 
 extern const char *nerv_chunk_file_tname;
 extern const char *nerv_chunk_file_handle_tname;
 extern const char *nerv_chunk_info_tname;
 extern const char *nerv_chunk_data_tname;
 
-typedef struct ChunkFileHandle {
-    FILE *fp;
-    int closed;
-} ChunkFileHandle;
-
-typedef struct ChunkInfo {
-    off_t offset, length;
-} ChunkInfo;
-
-typedef struct ChunkData {
-    FILE *fp;
-    char *data;
-} ChunkData;
-
 #endif
diff --git a/nerv/io/init.lua b/nerv/io/init.lua
index 647ff93..eb2e3e5 100644
--- a/nerv/io/init.lua
+++ b/nerv/io/init.lua
@@ -3,7 +3,7 @@ function nerv.ChunkFile:write_chunkdata(metadata, writer)
         nerv.error("metadata should be a Lua table")
         return
     end
-    return self:_write_chunkdata(table.tostring(metadata), writer)
+    return self._write_chunkdata(self.handle, table.tostring(metadata), writer)
 end
 
 function nerv.ChunkFile:write_chunk(chunk)
@@ -28,10 +28,14 @@ function nerv.ChunkFile:read_chunk(id, global_conf)
     local chunk_type = nerv.get_type(metadata.type)
     local chunk = chunk_type(id, global_conf)
     chunk:set_info(metadata.info)
-    chunk:read(self:get_chunkdata(id))
+    chunk:read(self._get_chunkdata(self.handle, metadata._chunk_info))
     return chunk
 end
 
+function nerv.ChunkFile:close()
+    self._close(self.handle)
+end
+
 local DataReader = nerv.class("nerv.DataReader")
 
 function DataReader:__init(global_conf, reader_conf)
diff --git a/nerv/lib/io/chunk_file.c b/nerv/lib/io/chunk_file.c
new file mode 100644
index 0000000..a305962
--- /dev/null
+++ b/nerv/lib/io/chunk_file.c
@@ -0,0 +1,239 @@
+#include "../../common.h"
+#include "chunk_file.h"
+#include <stdlib.h>
+#include <string.h>
+#define PARAM_HEADER_SIZE 16
+
+static size_t read_chunk_header_plain(FILE *fp, int *status) {
+    static char buff[PARAM_HEADER_SIZE];
+    int i;
+    size_t size = 0;
+    if (fread(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE)
+    {
+        if (feof(fp)) *status = CF_END_OF_FILE;
+        else
+        {
+            *status = CF_INVALID_FORMAT;
+            return 0;
+        }
+    }
+    else *status = CF_NORMAL;
+    for (i = 0; i < PARAM_HEADER_SIZE; i++)
+        if (isdigit(buff[i]))
+            size = size * 10 + buff[i] - '0';
+/*    fprintf(stderr, "header: %lu\n", size); */
+    return size;
+}
+
+static void write_chunk_header_plain(FILE *fp, size_t size, int *status) {
+    static char buff[PARAM_HEADER_SIZE];
+    int i;
+    for (i = PARAM_HEADER_SIZE - 3; i > 0; i--, size /= 10)
+        buff[i] = size % 10 + '0';
+    if (size)
+    {
+        *status = CF_SECTION_OVERFLOW;
+        return;
+    }
+    buff[0] = '[';
+    buff[PARAM_HEADER_SIZE - 2] = ']';
+    buff[PARAM_HEADER_SIZE - 1] = '\n';
+    if (fwrite(buff, 1, PARAM_HEADER_SIZE, fp) != PARAM_HEADER_SIZE)
+    {
+        *status = CF_WRITE_ERROR;
+        return;
+    }
+    *status = CF_NORMAL;
+}
+
+static ChunkData *get_chunk_data(FILE *fp, ChunkInfo *info) {
+    ChunkData *cdp = (ChunkData *)malloc(sizeof(ChunkData));
+    cdp->data = (char *)malloc(info->length);
+    cdp->fp = fmemopen(cdp->data, info->length, "r");
+    assert(fseeko(fp, info->offset, SEEK_SET) == 0);
+    if (fread(cdp->data, 1, info->length, fp) != (size_t)info->length)
+        return NULL;
+    return cdp;
+}
+
+static const char *read_chunk_metadata(FILE *fp, const char *fn, int *status) {
+#define LINEBUFF_SIZE 1024
+#define LUA_RETURN "return "
+#define LUA_RETURN_LEN (sizeof(LUA_RETURN) - 1)
+    static char buff[LUA_RETURN_LEN + LINEBUFF_SIZE] = LUA_RETURN;
+    *status = fgets(buff + LUA_RETURN_LEN,
+                    LINEBUFF_SIZE, fp) == (buff + LUA_RETURN_LEN) ? \
+                     CF_NORMAL : CF_INVALID_FORMAT;
+    fprintf(stderr, "metadata: %s\n", buff);
+    return buff;
+}
+
+static void write_chunk_metadata(FILE *fp, const char *mdstr, int *status) {
+    size_t size = strlen(mdstr);
+    if (fwrite(mdstr, 1, size, fp) != size ||
+        fprintf(fp, "\n") < 0)
+    {
+        *status = CF_WRITE_ERROR;
+        return;
+    }
+    /* fprintf(stderr, "metadata: %s\n", metadata_str); */
+    *status = CF_NORMAL;
+}
+
+static ChunkFile *open_write(const char *fn, int *status) {
+    ChunkFile *cfp;
+    FILE *fp = fopen(fn, "w");
+
+    if (!fp)
+    {
+        *status = CF_ERR_OPEN_FILE;
+        return NULL;
+    }
+    cfp = (ChunkFile *)malloc(sizeof(ChunkFile));
+    cfp->fp = fp;
+    cfp->status = CF_WRITE;
+    *status = CF_NORMAL;
+    return cfp;
+}
+
+static ChunkFile *open_read(const char *fn, int *status) {
+    size_t chunk_len;
+    off_t offset;
+    int i;
+    const char *mdstr;
+    ChunkFile *cfp;
+    ChunkInfo *head = NULL;
+    FILE *fp = fopen(fn, "r");
+
+    if (!fp)
+    {
+        *status = CF_ERR_OPEN_FILE;
+        return NULL;
+    }
+    cfp = (ChunkFile *)malloc(sizeof(ChunkFile));
+    cfp->fp = fp;
+    cfp->status = CF_READ;
+    offset = ftello(fp);
+    /* fprintf(stderr, "%d\n", (int)offset); */
+    for (i = 0;; offset += chunk_len, i++)
+    {
+        ChunkInfo *cip;
+        fprintf(stderr, "reading chunk %d from %d\n", i, (int)offset);
+        /* skip to the begining of chunk i */
+        if (fseeko(fp, offset, SEEK_SET) != 0)
+        {
+            *status = CF_INVALID_FORMAT;
+            return NULL;
+        }
+        /* read header */
+        chunk_len = read_chunk_header_plain(fp, status);
+        if (*status == CF_END_OF_FILE) break;
+        if (*status != CF_NORMAL)
+            return NULL;
+        cip = (ChunkInfo *)malloc(sizeof(ChunkInfo));
+        /* read metadata */
+        mdstr = read_chunk_metadata(fp, fn, status);
+        if (*status != CF_NORMAL)
+            return NULL;
+        cip->metadata = strdup(mdstr);
+        cip->offset = ftello(fp);
+        cip->length = chunk_len - (cip->offset - offset);
+        /* fprintf(stderr, "%d + %d (skip %lu)\n", (int)cip->offset,
+                (int)cip->length, chunk_len); */
+        cip->next = head;
+        head = cip;
+    }
+    *status = CF_NORMAL;
+    cfp->info = head;
+    return cfp;
+}
+
+ChunkFile *nerv_chunk_file_create(const char *fn, const char *mode, int *status) {
+    int rd = 1, bin = 0;
+    size_t i, len = strlen(mode);
+    for (i = 0; i < len; i++)
+        switch (mode[i])
+        {
+            case 'r': rd = 1; break;
+            case 'w': rd = 0; break;
+            case 'b': bin = 1; break;
+        }
+    return rd ? open_read(fn, status) : \
+                open_write(fn, status);
+}
+
+int nerv_chunk_file_write_chunkdata(ChunkFile *cfp, const char *mdstr,
+                                    ChunkDataWriter_t writer, void *writer_arg) {
+    int status;
+    off_t start;
+    size_t size;
+    if (cfp->status != CF_WRITE)
+        return CF_INVALID_OP;
+    start = ftello(cfp->fp);
+    write_chunk_header_plain(cfp->fp, 0, &status); /* fill zeros */
+    if (status != CF_NORMAL) return status;
+    write_chunk_metadata(cfp->fp, mdstr, &status);
+    if (status != CF_NORMAL) return status;
+    writer(writer_arg);
+    size = ftello(cfp->fp) - start;
+    fseeko(cfp->fp, start, SEEK_SET);
+    /* write the calced size */
+    write_chunk_header_plain(cfp->fp, size, &status);
+    if (status != CF_NORMAL) return status;
+    fseeko(cfp->fp, 0, SEEK_END);
+    return CF_NORMAL;
+}
+
+ChunkData *nerv_chunk_file_get_chunkdata(ChunkFile *cfp, ChunkInfo *cip, int *status) {
+    ChunkData *cdp;
+    if (cfp->status != CF_READ)
+    {
+        *status = CF_INVALID_OP;
+        return NULL;
+    }
+    if (!(cdp = get_chunk_data(cfp->fp, cip)))
+    {
+        *status = CF_END_OF_FILE;
+        return NULL;
+    }
+    *status = CF_NORMAL;
+    return cdp;
+}
+
+void nerv_chunk_file_close(ChunkFile *cfp) {
+    if (cfp->status != CF_CLOSED)
+        fclose(cfp->fp);
+    cfp->status = CF_CLOSED;
+}
+
+void nerv_chunk_file_destroy(ChunkFile *cfp) {
+    ChunkInfo *i, *ni;
+    if (cfp->status != CF_CLOSED) fclose(cfp->fp);
+    for (i = cfp->info; i; i = ni)
+    {
+        ni = i->next;
+        free(i->metadata);
+        free(i);
+    }
+    free(cfp);
+}
+
+void nerv_chunk_data_destroy(ChunkData *cdp) {
+    fclose(cdp->fp);
+    free(cdp->data);
+    free(cdp);
+}
+
+const char *nerv_chunk_file_errstr(int status) {
+    switch (status)
+    {
+        case CF_INVALID_FORMAT: return "invalid format";
+        case CF_END_OF_FILE: return "unexpected end of file";
+        case CF_SECTION_OVERFLOW: return "section overflow";
+        case CF_WRITE_ERROR: return "error while writing";
+        case CF_ERR_OPEN_FILE: return "error while opening file";
+        case CF_INVALID_OP: return "invalid operation";
+        default: return "unknown";
+    }
+    return NULL;
+}
diff --git a/nerv/lib/io/chunk_file.h b/nerv/lib/io/chunk_file.h
new file mode 100644
index 0000000..71f0d03
--- /dev/null
+++ b/nerv/lib/io/chunk_file.h
@@ -0,0 +1,43 @@
+#ifndef NERV_CHUNK_FILE_H
+#define NERV_CHUNK_FILE_H
+#include "../../common.h"
+enum {
+    CF_NORMAL,
+    CF_INVALID_FORMAT,
+    CF_END_OF_FILE,
+    CF_SECTION_OVERFLOW,
+    CF_WRITE_ERROR,
+    CF_ERR_OPEN_FILE,
+    CF_INVALID_OP,
+    CF_READ,
+    CF_WRITE,
+    CF_CLOSED
+};
+
+typedef struct ChunkInfo {
+    struct ChunkInfo *next;
+    char *metadata;
+    off_t offset, length;
+} ChunkInfo;
+
+typedef struct ChunkFile {
+    FILE *fp;
+    ChunkInfo *info;
+    int status;
+} ChunkFile;
+
+typedef struct ChunkData {
+    FILE *fp;
+    char *data;
+} ChunkData;
+
+typedef void (*ChunkDataWriter_t)(void *);
+ChunkFile *nerv_chunk_file_create(const char *fn, const char *mode, int *status);
+int nerv_chunk_file_write_chunkdata(ChunkFile *cfp, const char *mdstr,
+                                    ChunkDataWriter_t writer, void *writer_arg);
+ChunkData *nerv_chunk_file_get_chunkdata(ChunkFile *cfp, ChunkInfo *cip, int *status);
+void nerv_chunk_file_close(ChunkFile *cfp);
+void nerv_chunk_file_destroy(ChunkFile *cfp);
+void nerv_chunk_data_destroy(ChunkData *cdp);
+const char *nerv_chunk_file_errstr(int status);
+#endif
diff --git a/nerv/luaT/README.md b/nerv/lib/luaT/README.md
index 6e9cf0d..6e9cf0d 100644
--- a/nerv/luaT/README.md
+++ b/nerv/lib/luaT/README.md
diff --git a/nerv/luaT/luaT.c b/nerv/lib/luaT/luaT.c
index 7b85ce3..7b85ce3 100644
--- a/nerv/luaT/luaT.c
+++ b/nerv/lib/luaT/luaT.c
diff --git a/nerv/luaT/luaT.h b/nerv/lib/luaT/luaT.h
index 5e8dd2f..5e8dd2f 100644
--- a/nerv/luaT/luaT.h
+++ b/nerv/lib/luaT/luaT.h
diff --git a/nerv/matrix/generic/mmatrix.c b/nerv/matrix/generic/mmatrix.c
index b0f0791..697c9fc 100644
--- a/nerv/matrix/generic/mmatrix.c
+++ b/nerv/matrix/generic/mmatrix.c
@@ -51,11 +51,11 @@ static void host_matrix_(init)(lua_State *L) {
 #include "matrix.c"
 
 int nerv_matrix_(load)(lua_State *L) {
-    ChunkData *chunk = luaT_checkudata(L, 1, nerv_chunk_data_tname);
+    ChunkData *cdp = luaT_checkudata(L, 1, nerv_chunk_data_tname);
     Matrix *self;
     int i, j;
     long nrow, ncol;
-    FILE *fp = chunk->fp;
+    FILE *fp = cdp->fp;
     if (fscanf(fp, "%ld %ld", &nrow, &ncol) != 2)
         return 0;
     self = nerv_matrix_(new_)(L, nrow, ncol);
@@ -74,12 +74,12 @@ int nerv_matrix_(load)(lua_State *L) {
 }
 
 int nerv_matrix_(save)(lua_State *L) {
-    ChunkFileHandle *chunk = luaT_checkudata(L, 2,
-                                nerv_chunk_file_handle_tname);
+    ChunkFile *cfp = luaT_checkudata(L, 2,
+                            nerv_chunk_file_handle_tname);
     Matrix *self = luaT_checkudata(L, 1, nerv_matrix_(tname));
     int i, j;
     long nrow = self->nrow, ncol = self->ncol;
-    FILE *fp = chunk->fp;
+    FILE *fp = cfp->fp;
     if (fprintf(fp, "%ld %ld\n", nrow, ncol) < 0)
         return 0;
     for (i = 0; i < nrow; i++)
diff --git a/nerv/nerv-scm-1.rockspec b/nerv/nerv-scm-1.rockspec
index d14140a..0b7e4cb 100644
--- a/nerv/nerv-scm-1.rockspec
+++ b/nerv/nerv-scm-1.rockspec
@@ -1,21 +1,19 @@
 package = "nerv"
 version = "scm-1"
 source = {
-    url = "..." -- We don't have one yet
+    url = "https://github.com/Determinant/nerv.git"
 }
 description = {
-    summary = "An example for the LuaRocks tutorial.",
+    summary = "A Lua-based toolkit dedicated to high-performance deep neural network learning",
     detailed = [[
     ]],
-    homepage = "https://github.com/Determinant/nerv", -- We don't have one yet
-    license = "BSD" -- or whatever you like
+    homepage = "https://github.com/Determinant/nerv",
+    license = "BSD"
 }
 dependencies = {
     "lua >= 5.1"
-    -- If you depend on other rocks, add them here
 }
 build = {
-    -- We'll start here.
     type = "make",
     build_variables = {
         CFLAGS="$(CFLAGS)",