diff options
Diffstat (limited to 'nerv')
59 files changed, 6468 insertions, 2 deletions
@@ -1,2 +0,0 @@ -#!/bin/bash -exec 'build/luajit-2.0/bin/luajit' -e "package.cpath=\"${PWD}/build/lib/?.so\"" -e "package.path=\"${PWD}/build/lua/?/init.lua;${PWD}/build/lua/?.lua;${PWD}/?.lua\"" -e "require 'nerv'" "$@" diff --git a/nerv/.gitignore b/nerv/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/nerv/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/nerv/Makefile b/nerv/Makefile new file mode 100644 index 0000000..b69a63e --- /dev/null +++ b/nerv/Makefile @@ -0,0 +1,60 @@ +.PHONY: build install clean +SHELL := /bin/bash +BUILD_DIR := $(CURDIR)/build +OBJS := nerv.o luaT.o common.o \ + matrix/mmatrix.o matrix/cumatrix.o matrix/init.o matrix/cukernel.o \ + io/init.o io/chunk_file.o \ + examples/oop_example.o +LIBS := libnerv.so +LUA_LIBS := matrix/init.lua io/init.lua init.lua \ + layer/init.lua layer/affine.lua layer/sigmoid.lua layer/softmax_ce.lua \ + layer/window.lua layer/bias.lua layer/combiner.lua layer/mse.lua \ + nn/init.lua nn/layer_repo.lua nn/param_repo.lua nn/layer_dag.lua \ + io/sgd_buffer.lua +INCLUDE := -I $(LUA_INCDIR) -DLUA_USE_APICHECK +CUDA_BASE := /usr/local/cuda-6.5 +#CUDA_BASE := /usr/local/cuda-5.0 +CUDA_INCLUDE := -I $(CUDA_BASE)/include/ +INCLUDE += $(CUDA_INCLUDE) +LDFLAGS := -L$(CUDA_BASE)/lib64/ -Wl,-rpath=$(CUDA_BASE)/lib64/ -lcudart -lcublas +CFLAGS := -Wall -Wextra -O2 +OBJ_DIR := $(BUILD_DIR)/objs +SUBDIR := matrix io layer examples nn +NVCC := $(CUDA_BASE)/bin/nvcc +NVCC_FLAGS := -Xcompiler -fPIC,-Wall,-Wextra + +LUA_DIR = $(INST_LUADIR)/nerv +OBJS := $(addprefix $(OBJ_DIR)/,$(OBJS)) +OBJ_SUBDIR := $(addprefix $(OBJ_DIR)/,$(SUBDIR)) +LUA_SUBDIR := $(addprefix $(LUA_DIR)/,$(SUBDIR)) +LIBS := $(addprefix $(INST_LIBDIR)/,$(LIBS)) +LUA_LIBS := $(addprefix $(LUA_DIR)/,$(LUA_LIBS)) + +build: $(OBJ_DIR) $(OBJ_SUBDIR) $(OBJS) +$(OBJ_DIR) $(LUA_DIR) $(OBJ_SUBDIR) $(LUA_SUBDIR): + -mkdir -p $@ +$(OBJ_DIR)/%.o: %.c $(patsubst /%.o,/%.c,$@) + gcc -c -o $@ $< $(INCLUDE) -fPIC $(CFLAGS) +$(OBJ_DIR)/matrix/cukernel.o: matrix/cukernel.cu + $(NVCC) -c -o $@ $< $(INCLUDE) $(NVCC_FLAGS) +$(LUA_DIR)/%.lua: %.lua + cp $< $@ +$(OBJ_DIR)/luaT.o: + gcc -c -o $@ luaT/luaT.c $(INCLUDE) -fPIC +$(LIBS): $(OBJS) + gcc -shared -o $@ $(OBJS) $(LDFLAGS) + +$(OBJ_DIR)/matrix/cumatrix.o: matrix/generic/cumatrix.c matrix/generic/matrix.c matrix/generic/cukernel.cu +$(OBJ_DIR)/matrix/mmatrix.o: matrix/generic/mmatrix.c matrix/generic/matrix.c +$(OBJ_DIR)/matrix/cukernel.o: matrix/generic/cukernel.cu + +.PHONY: speech + +speech: + -mkdir -p build/objs/speech/tnet_io + $(MAKE) -C speech/ BUILD_DIR=$(BUILD_DIR) LIB_DIR=$(LIB_DIR) OBJ_DIR=$(CURDIR)/build/objs/speech/ LUA_DIR=$(LUA_DIR) + +clean: + -rm -rf $(OBJ_DIR) + +install: $(LIBS) $(LUA_DIR) $(LUA_SUBDIR) $(LUA_LIBS) diff --git a/nerv/common.c b/nerv/common.c new file mode 100644 index 0000000..b4e39e6 --- /dev/null +++ b/nerv/common.c @@ -0,0 +1,76 @@ +#include "common.h" +#include <stdarg.h> +int nerv_error(lua_State *L, const char *err_mesg_fmt, ...) { + va_list ap; + va_start(ap, err_mesg_fmt); + lua_pushstring(L, "[nerv] internal error: "); + lua_pushvfstring(L, err_mesg_fmt, ap); + lua_concat(L, 2); + lua_error(L); + va_end(ap); + return 0; +} + +int nerv_error_method_not_implemented(lua_State *L) { + return nerv_error(L, "method not implemented"); +} + +void luaN_append_methods(lua_State *L, const luaL_Reg *mlist) { + for (; mlist->func; mlist++) + { + lua_pushcfunction(L, mlist->func); + lua_setfield(L, -2, mlist->name); + } +} + +HashMap *hashmap_create(size_t size, HashKey_t hfunc, HashMapCmp_t cmp) { + HashMap *res = (HashMap *)malloc(sizeof(HashMap)); + res->bucket = calloc(size, sizeof(HashNode)); + res->cmp = cmp; + res->hfunc = hfunc; + res->size = size; + return res; +} + +void *hashmap_getval(HashMap *h, const char *key) { + size_t idx = h->hfunc(key) % h->size; + HashNode *ptr; + for (ptr = h->bucket[idx]; ptr; ptr = ptr->next) + { + if (!h->cmp(ptr->key, key)) + return ptr->val; + } + return NULL; +} + +void hashmap_setval(HashMap *h, const char *key, void *val) { + size_t idx = h->hfunc(key) % h->size; + HashNode *ptr = malloc(sizeof(HashNode)); + ptr->next = h->bucket[idx]; + h->bucket[idx] = ptr; + ptr->key = key; + ptr->val = val; +} + +void hashmap_clear(HashMap *h) { + size_t i; + for (i = 0; i < h->size; i++) + { + HashNode *ptr, *nptr; + for (ptr = h->bucket[i]; ptr; ptr = nptr) + { + nptr = ptr->next; + free(ptr->val); + free(ptr); + } + h->bucket[i] = NULL; + } +} + +size_t bkdr_hash(const char *key) { + unsigned int seed = 131; + unsigned int res = 0; + while (*key) + res = res * seed + *key++; + return res; +} diff --git a/nerv/common.h b/nerv/common.h new file mode 100644 index 0000000..e21c7a5 --- /dev/null +++ b/nerv/common.h @@ -0,0 +1,36 @@ +#ifndef NERV_COMMON_H +#define NERV_COMMON_H +#include "lua.h" +#include "lauxlib.h" +#include "lualib.h" +#include "luaT/luaT.h" +#include <stdio.h> +#include <stdlib.h> + +typedef struct HashNode { + const char *key; + void *val; + struct HashNode *next; +} HashNode; + +typedef int (*HashMapCmp_t)(const char *a, const char *b); +typedef size_t (*HashKey_t)(const char *key); + +typedef struct HashMap { + HashNode **bucket; + HashMapCmp_t cmp; + HashKey_t hfunc; + size_t size; +} HashMap; + +HashMap *hashmap_create(size_t size, HashKey_t hfunc, HashMapCmp_t cmp); +void *hashmap_getval(HashMap *h, const char *key); +void hashmap_setval(HashMap *h, const char *key, void *val); +void hashmap_clear(HashMap *h); + +size_t bkdr_hash(const char *key); + +int nerv_error(lua_State *L, const char *err_mesg_fmt, ...); +int nerv_error_method_not_implemented(lua_State *L); +void luaN_append_methods(lua_State *L, const luaL_Reg *mlist); +#endif diff --git a/nerv/doc/nerv.md b/nerv/doc/nerv.md new file mode 100644 index 0000000..28411f5 --- /dev/null +++ b/nerv/doc/nerv.md @@ -0,0 +1,17 @@ +#The Nerv utility functions# +Part of the [Nerv](../README.md) toolkit. +##Methods## +* __string = nerv.typename(obj a)__ +A registered function, the original function is `luaT_lua_typename`. In some cases if you call `type(a)` for object of some class in __Nerv__(like __Nerv.CuMatrix__) it will only return "userdata"(because it is created in C), in this case you can use this method to get its type. + +--- + +* __metatable = nerv.getmetatable(string tname)__ +A registered function, the original function is `luaT_lua_getmetatable`. `tname` should be a class name that has been registered in __luaT__. + +* __metatable = nerv.newmetatable(string tname, string parenttname, function constructor, function destructor, function factory)__ +A registered function, the original function is `luaT_newmetatable`, it returns the metatable of the created class by the name `tname`. +* __string = nerv.setmetatable(table self, string tname)__ +A registered function, the original function is `luaT_lua_setmetatable`. It assigns the metatable registered in __luaT__ by the name *tname* to the table *self*. And return *tname* to user. +* __table = nerv.get_type(string typename)__ +Returns the type(`loadstring("return " .. typename)`).
\ No newline at end of file diff --git a/nerv/doc/nerv_class.md b/nerv/doc/nerv_class.md new file mode 100644 index 0000000..99f63e7 --- /dev/null +++ b/nerv/doc/nerv_class.md @@ -0,0 +1,36 @@ +#The Nerv OOP# +Part of the [Nerv](../README.md) toolkit. +##Methods## +* __metatable mt, metatable mpt = nerv.class(string tname, string parenttname)__ +This method is used to create a class by the name `tname`, which inherits `parenttname` in __Nerv__, then you create a new instance of this class by calling `obj=tname(...)`. The `tname.__init(...)` method(if defined) will be called in the constructing. The metatable of the class and its parent class will be returned. + +##Examples## +* This example implements a simple `nerv.Counter` class which is inherited by `nerv.BetterCounter`. + +``` +do + nerv.class("nerv.Counter") + function nerv.Counter:__init(c) + if (c) then + self.c = c + else + self.c = 0 + end + end +end +do + local mt, mpt = nerv.class("nerv.BetterCounter", "nerv.Counter") + function nerv.BetterCounter:__init(c, bc) + mpt.__init(self, c) + if (bc) then + self.bc = bc + else + self.bc = 0 + end + end +end +c1 = nerv.Counter(1) +print(c1.c) +bc1 = nerv.BetterCounter(1, 1) +print(bc1.c, bc1.bc) +```
\ No newline at end of file diff --git a/nerv/doc/nerv_io.md b/nerv/doc/nerv_io.md new file mode 100644 index 0000000..07589df --- /dev/null +++ b/nerv/doc/nerv_io.md @@ -0,0 +1,113 @@ +#The Nerv IO Package# +Part of the [Nerv](../README.md) toolkit. + +##Description## +The main class that the user uses to store and read parameter object to and from files is __nerv.ChunkFile__. +In the file, a parameter object will be saved using a standard format. First is the length(in byte) of this object, then a table which includes some meta information of the object, and a data area. Below is an example text file. +``` +[0000000000202] +{type="nerv.ExampleP",info={message="just-a-try"},id="exampleP1"} +3 3 +5.000000 5.000000 5.000000 +5.000000 5.000000 5.000000 +5.000000 5.000000 5.000000 +1 3 +4.000000 4.000000 4.000000 +[0000000000202] +{type="nerv.ExampleP",info={message="just-a-try"},id="exampleP2"} +3 3 +4.000000 4.000000 4.000000 +4.000000 4.000000 4.000000 +4.000000 4.000000 4.000000 +1 3 +3.000000 3.000000 3.000000 +``` + +##Methods## +* __ChunkFile ChunkFile(string fn, string mode)__ +`mode` can be `r` or `w`, for reading or writing a file. The returned __ChunkFile__ will be ready to write or read objects which follows the __nerv.Param__ interface(using `write_chunk` and `read_chunk`). +* __void ChunkFile.write_chunk(ChunkFile self, Param p)__ +Write `p` into the file. `p:write` will be called. +* __Param ChunkFile.read_chunk(ChunkFile self, string id, table global_conf)__ +Read the __Param__ object by id `id` from the file `self`. It will be constructed using `__init(id, global_conf)`. `p:read` will be called. +* __void ChunkFile.close(ChunkFile self)__ +Close the opened file. + +##Examples## +* An example showing how to use __ChunkFile__ to store and read parameter objects. +``` +require 'io' +do + local mt, mpt = nerv.class('nerv.ExampleP', 'nerv.Param') + function nerv.ExampleP:__init(id, global_conf) + self.id = id + self.global_conf = global_conf + self.matrix = nerv.MMatrixFloat(3, 3) + for i = 0, 2, 1 do + for j = 0, 2, 1 do + self.matrix[i][j] = 3 + end + end + self.bias = nerv.MMatrixFloat(1, 3) + for i = 0, 2, 1 do + self.bias[i] = 2; + end + self:set_info({message = 'just-a-try'}) + end + function nerv.ExampleP:addOne() + for i = 0, 2, 1 do + for j = 0, 2, 1 do + self.matrix[i][j] = self.matrix[i][j] + 1 + end + end + for i = 0, 2, 1 do + self.bias[i] = self.bias[i] + 1 + end + end + function nerv.ExampleP:read(pcdata) + self.matrix = nerv.MMatrixFloat.load(pcdata) + self.bias = nerv.MMatrixFloat.load(pcdata) + end + function nerv.ExampleP:write(pfhandle) + self.matrix:save(pfhandle) + self.bias:save(pfhandle) + end +end +global_conf = {} +do + local f = nerv.ChunkFile('../tmp', 'w') + local exampleP1 = nerv.ExampleP('exampleP1', global_conf) + local exampleP2 = nerv.ExampleP('exampleP2', global_conf) + exampleP1:addOne() + exampleP1:addOne() + exampleP2:addOne() + + f:write_chunk(exampleP1) + f:write_chunk(exampleP2) + f:close() +end +do + local f = nerv.ChunkFile('../tmp', 'r') + local exampleP1 = f:read_chunk('exampleP1', global_conf) + local exampleP2 = f:read_chunk('exampleP2', global_conf) + f:close() + print(exampleP1.matrix) + print(exampleP2.matrix) +end +``` + +##Developer Notes## +* There are four classes in to deal with chunk data, which are __nerv.ChunkFile__, __nerv.ChunkFileHandle__, __nerv.ChunkInfo__, __nerv.ChunkData__. Below is the underlying C structs. +``` +typedef struct ChunkFileHandle { + FILE *fp; +} ChunkFileHandle; +typedef struct ChunkInfo { + off_t offset, length; +} ChunkInfo; +typedef struct ChunkData { + FILE *fp; + char *data; +} ChunkData; +``` +* In __Nerv.io__, a returned(by `ChunkFile.__init`) __nerv.ChunkFile__ will have a member `handle`, which is a __nerv.ChunkFileHandle__.
\ No newline at end of file diff --git a/nerv/doc/nerv_layer.md b/nerv/doc/nerv_layer.md new file mode 100644 index 0000000..de2fb12 --- /dev/null +++ b/nerv/doc/nerv_layer.md @@ -0,0 +1,180 @@ +#The Nerv Layer Package# +Part of the [Nerv](../README.md) toolkit. + +##Description## +__nerv.Layer__ is the base class and most of its methods are abstract. +###Class hierarchy and their members### +* __nerv.Layer__. + * `table dim_in` It specifies the dimensions of the inputs. + * `table dim_out` It specifies the dimensions of the outputs. + * `string id` ID of this layer. + * `table gconf` Stores the `global_conf`. +* __nerv.AffineLayer__ inherits __nerv.Layer__, both `#dim_in` and `#dim_out` are 1. + * `MatrixParam ltp` The liner transform parameter. + * `BiasParam bp` The bias parameter. +* __nerv.BiasLayer__ inherits __nerv.Layer__, both `#dim_in` nad `#dim_out` are 1. + * `BiasParam bias` The bias parameter. +* __nerv.SigmoidLayer__ inherits __nerv.Layer__, both `#dim_in` and `#dim_out` are 1. +* __nerv.SoftmaxCELayer__ inherits __nerv.Layer__, `#dim_in` is 2 and `#dim_out` is -1(optional). `input[1]` is the input to the softmax layer, `input[2]` is the reference distribution. In its `propagate(input, output)` method, if `output[1] ~= nil`, cross\_entropy value will outputed. + * `float total_ce` Records the accumlated cross entropy value. + * `int total_frams` Records how many frames have passed. + * `bool compressed` The reference distribution can be a one-hot format. This feature is enabled by `layer_conf.compressed`. + +##Methods## +* __void Layer.\_\_init(Layer self, string id, table global_conf, table layer_conf)__ +Abstract method. +The constructing method should assign `id` to `self.id` and `global_conf` to `self.gconf`, `layer_conf.dim_in` to `self.dim_in`, `layer_conf.dim_out` to `self.dim_out`. `dim_in` and `dim_out` are a list specifies the dimensions of the inputs and outputs. Also, `layer_conf` will include the parameters, which should also be properly saved. +* __void Layer.init(Layer self)__ +Abstract method. +Initialization method, in this method the layer should do some self-checking and allocate space for intermediate results. +* __void Layer.update(Layer self, table bp_err, table input, table output)__ +Abstract method. +`bp_err[i]` should be the error on `output[i]`. In this method the parameters of `self` is updated. +* __void Layer.propagate(Layer self, table input, table output)__ +Abstract method. +Given `input` and the current parameters, propagate and store the result in `output`. +* __void Layer.back_propagate(Layer self, Matrix next_bp_err, Matrix bp_err, Matrix input, Matrix output)__ +Abstract method. +Calculate the error on the inputs and store them in `next_bp_err`. + +* __void Layer.check_dim_len(int len_in, int len_out)__ +Check whether `#self.dim_in == len_in` and `#self.dim_out == len_out`, if violated, an error will be posted. +* __void Layer.get_params(Layer self)__ +Abstract method. +The layer should return a list containing its parameters. + +####nerv.Layer.get\_dim(self)#### +* Returns: + `dim_in`: __table__. + `dim_out`: __table__. +* Parameters: + `self`: __nerv.Layer__. +* Description: + Returns `self.dim_in, self.dim_out`. + +##Examples## +* a basic example using __Nerv__ layers to a linear classification. + +``` +require 'math' + +require 'layer.affine' +require 'layer.softmax_ce' + +--[[Example using layers, a simple two-classification problem]]-- + +function calculate_accurate(networkO, labelM) + sum = 0 + for i = 0, networkO:nrow() - 1, 1 do + if (labelM[i][0] == 1 and networkO[i][0] >= 0.5) then + sum = sum + 1 + end + if (labelM[i][1] == 1 and networkO[i][1] >= 0.5) then + sum = sum + 1 + end + end + return sum +end + +--[[begin global setting and data generation]]-- +global_conf = {lrate = 10, + wcost = 1e-6, + momentum = 0.9, + cumat_type = nerv.CuMatrixFloat} + +input_dim = 5 +data_num = 100 +ansV = nerv.CuMatrixFloat(input_dim, 1) +for i = 0, input_dim - 1, 1 do + ansV[i][0] = math.random() - 0.5 +end +ansB = math.random() - 0.5 +print('displaying ansV') +print(ansV) +print('displaying ansB(bias)') +print(ansB) + +dataM = nerv.CuMatrixFloat(data_num, input_dim) +for i = 0, data_num - 1, 1 do + for j = 0, input_dim - 1, 1 do + dataM[i][j] = math.random() * 2 - 1 + end +end +refM = nerv.CuMatrixFloat(data_num, 1) +refM:fill(ansB) +refM:mul(dataM, ansV, 1, 1) --refM = dataM * ansV + ansB + +labelM = nerv.CuMatrixFloat(data_num, 2) +for i = 0, data_num - 1, 1 do + if (refM[i][0] > 0) then + labelM[i][0] = 1 + labelM[i][1] = 0 + else + labelM[i][0] = 0 + labelM[i][1] = 1 + end +end +--[[global setting and data generation end]]-- + + +--[[begin network building]]-- +--parameters +affineL_ltp = nerv.LinearTransParam('AffineL_ltp', global_conf) +affineL_ltp.trans = nerv.CuMatrixFloat(input_dim, 2) +for i = 0, input_dim - 1, 1 do + for j = 0, 1, 1 do + affineL_ltp.trans[i][j] = math.random() - 0.5 + end +end +affineL_bp = nerv.BiasParam('AffineL_bp', global_conf) +affineL_bp.trans = nerv.CuMatrixFloat(1, 2) +for j = 0, 1, 1 do + affineL_bp.trans[j] = math.random() - 0.5 +end + +--layers +affineL = nerv.AffineLayer('AffineL', global_conf, {['ltp'] = affineL_ltp, + ['bp'] = affineL_bp, + dim_in = {input_dim}, + dim_out = {2}}) +softmaxL = nerv.SoftmaxCELayer('softmaxL', global_conf, {dim_in = {2, 2}, + dim_out = {}}) +print('layers initializing...') +affineL:init() +softmaxL:init() +--[[network building end]]-- + + +--[[begin space allocation]]-- +print('network input&output&error space allocation...') +affineI = {dataM} --input to the network is data +affineO = {nerv.CuMatrixFloat(data_num, 2)} +softmaxI = {affineO[1], labelM} +softmaxO = {} +output = nerv.CuMatrixFloat(data_num, 2) + +affineE = {nerv.CuMatrixFloat(data_num, 2)} +--[[space allocation end]]-- + + +--[[begin training]]-- +ce_last = 0 +for l = 0, 10, 1 do + affineL:propagate(affineI, affineO) + softmaxL:propagate(softmaxI, softmaxO) + output:softmax(softmaxI[1]) + + softmaxL:back_propagate(affineE, {}, softmaxI, softmaxO) + + affineL:update(affineE, affineI, affineO) + + if (l % 5 == 0) then + nerv.utils.printf("training iteration %d finished\n", l) + nerv.utils.printf("cross entropy: %.8f\n", softmaxL.total_ce - ce_last) + ce_last = softmaxL.total_ce + nerv.utils.printf("accurate labels: %d\n", calculate_accurate(output, labelM)) + nerv.utils.printf("total frames processed: %.8f\n", softmaxL.total_frames) + end +end +--[[end training]]-- +``` diff --git a/nerv/doc/nerv_matrix.md b/nerv/doc/nerv_matrix.md new file mode 100644 index 0000000..22971d2 --- /dev/null +++ b/nerv/doc/nerv_matrix.md @@ -0,0 +1,165 @@ +#The Nerv Matrix Package# +Part of the [Nerv](../README.md) toolkit. + +##Description## +###Underlying structure### +In the begining is could be useful to know something about the underlying structure of a __Nerv__ matrix. Please keep in mind that matrice in __Nerv__ is row-major. +Every matrix object is a encapsulation of a C struct that describes the attributes of this matrix. +``` +typedef struct Matrix { + size_t stride; /* size of a row */ + long ncol, nrow, nmax; /* dimension of the matrix, nmax is simply nrow * ncol */ + union { + float *f; + |