diff options
Diffstat (limited to 'nerv/layer')
| -rw-r--r-- | nerv/layer/affine.lua | 39 | ||||
| -rw-r--r-- | nerv/layer/graph.lua | 63 | ||||
| -rw-r--r-- | nerv/layer/init.lua | 156 |
3 files changed, 237 insertions, 21 deletions
diff --git a/nerv/layer/affine.lua b/nerv/layer/affine.lua index a1c92b1..2dd2dc0 100644 --- a/nerv/layer/affine.lua +++ b/nerv/layer/affine.lua @@ -1,24 +1,38 @@ ---- Parameter and layer classes related to linear transform. +--- Contains parameter and layer classes related to linear (or affine) +-- transform. + +--- The class for linear transform parameter. +-- @type nerv.LinearTransParam -local MatrixParam = nerv.class('nerv.MatrixParam', 'nerv.Param') local LinearTransParam = nerv.class('nerv.LinearTransParam', 'nerv.MatrixParam') + +--- The class for bias parameter (currently implemented as a one-row matrix). +-- @type nerv.BiasParam + local BiasParam = nerv.class('nerv.BiasParam', 'nerv.MatrixParam') -local AffineLayer = nerv.class('nerv.AffineLayer', 'nerv.Layer') ---- A parameter that consists of a single matrix +--- The class for all matrix-based parameters. The class has a single matrix +-- which can be accessed by `self.trans`. -- @type nerv.MatrixParam +local MatrixParam = nerv.class('nerv.MatrixParam', 'nerv.Param') + +--- Check the storage location of the contained matrix. This function is +-- required by `nerv.ParamRepo`. +-- @param checker the callback function for checking function MatrixParam:check(checker) -- check trans matrix type checker(self.trans) end ---- Read from a file handle. +--- Read from a file handle. See `nerv.Param.read`. -- @param handle the file handle function MatrixParam:read(handle) self.trans = self.gconf.mmat_type.load(handle) end +--- Write to a file handle. See `nerv.Param.write`. +-- @param handle the file handle function MatrixParam:write(handle) self.trans:save(handle) end @@ -69,10 +83,23 @@ function MatrixParam:update_by_err_input() self:_update(l2, l2) end ---- A fully-connected linear transform layer. +--- The affine layer that does the calculation Wx + b, also known as fully +-- connected linear transform layer. -- @type nerv.AffineLayer +local AffineLayer = nerv.class('nerv.AffineLayer', 'nerv.Layer') + --- The constructor. +-- @param id the identifier +-- @param global_conf see `self.gconf` of `nerv.Layer.__init` +-- @param layer_conf a table providing with settings dedicated for the layer, +-- for `layer_conf` fields that are shared by all layers, see +-- `nerv.Layer.__init`. The affine layer requires parameters to be bound, the +-- following parameter names will be looked up while binding: +-- +-- * `ltp`: the linear transformation parameter, also known as the weight matrix, W in Wx + b +-- * `bp`: the bias parameter, also known as the bias matrix, b in Wx + b + function AffineLayer:__init(id, global_conf, layer_conf) nerv.Layer.__init(self, id, global_conf, layer_conf) self:check_dim_len(-1, 1) -- exactly one output, allow multiple inputs diff --git a/nerv/layer/graph.lua b/nerv/layer/graph.lua index 5790f95..5b5d4c7 100644 --- a/nerv/layer/graph.lua +++ b/nerv/layer/graph.lua @@ -1,5 +1,68 @@ +--- Implements a special kind of layers having an internal structure, a +-- directed graph of connected sub-level layers. + +--- The class describing the concept of a graph layer having an internal +-- structure, a directed graph of connected sub-level layers. Some of these +-- sub-level layers can again be graph layers, thus, it enables nested and +-- recursive layer declaration. The graph layer can be regarded as a container of +-- its sub-level layers. A layer other than a graph layer is also referenced as +-- "*primitive layer*". +-- @type nerv.GraphLayer + local GraphLayer = nerv.class('nerv.GraphLayer', 'nerv.Layer') +--- The constructor. +-- @param id the identifier +-- @param global_conf see `self.gconf` of `nerv.Layer.__init` +-- @param layer_conf a table providing with settings dedicated for the layer, +-- the following fields should be specified: +-- +-- * `lrepo`: the layer repo that should be used to find the sub-level layers +-- * `connections`: an array of 3-tuples describing the connections of +-- sub-level layers, the structure is as follow: +-- +-- { +-- {<from_port1>, <to_port1>, <time_shift1>}, -- tuple 1 +-- {<from_port2>, <to_port2>, <time_shift2>}, -- tuple 2 +-- {<from_port3>, <to_port3>, <time_shift3>}, -- tuple 3 +-- ... +-- } +-- Each tuple stands for a directed edge between two ports. The first two +-- elements in the tuple are called *port specification* which is a string +-- with the following format: +-- +-- <layer_id>[<port_idx>] +-- where the `<layer_id>` is a string that identifies the layer in +-- `lconf.lrepo`, and `<port_id>` is the input or output port index when used +-- in the first or second port specification respectively. +-- +-- The third element in the tuple is an integer specifying the time delay of +-- this connection. In most cases, it will be simply zero. But for an +-- recurrent network, a positive value `i` means the output from `<from_port>` +-- will be used as the input to `<to_port>` in `i`th computation of the future. +-- Negative values are also allowed to propagate the output to the past. +-- +-- Note that there are two possible strings of `<layer_id>` that have special +-- meanings: the string `"<input>"` and `"<output>"` are placeholders of the +-- the input and output ports of the outer graph layer. The input for the graph +-- layer as a whole can be used by establishing connections from +-- `"<input>[i]"`, and vice versa for the output. +-- +-- As an example, tuples: +-- +-- { +-- {"<input>[1]", "affine0[1]", 0}, +-- {"affine0[1]", "sigmoid0[1]", 0}, +-- {"sigmoid0[1]", "affine1[1]", 0}, +-- {"affine1[1]", "<output>[1]", 0} +-- } +-- Specify a graph layer that contains two stacked and fully connected linear +-- transformation sub-level layers. +-- +-- * `reversed`: optional, reverse the time shifting of all connections if true +-- +-- For other `layer_conf` fields that are shared by all layers, see `nerv.Layer.__init`. + function GraphLayer:__init(id, global_conf, layer_conf) nerv.Layer.__init(self, id, global_conf, layer_conf) self.lrepo = layer_conf.layer_repo diff --git a/nerv/layer/init.lua b/nerv/layer/init.lua index c893df3..7521b7a 100644 --- a/nerv/layer/init.lua +++ b/nerv/layer/init.lua @@ -1,38 +1,96 @@ --- The following methods must be implemented to let a layer work properly +--- Implements the concept of groups of parameters (`nerv.Param`) and +-- computation nodes (`nerv.Layer`). + +--- The class describing a group of parameters (an internal state) that can be +-- bound to layers. This class also implements the *chunk* interface (see +-- `nerv.ChunkFile`) , which means instances of `nerv.Param` can be exported to +-- chunk files as chunks. +-- @type nerv.Param local Param = nerv.class('nerv.Param') +--- The constructor. +-- @param id the identifier for the group of parameters +-- @param global_conf a table describing the computation state and providing +-- with some global settings + function Param:__init(id, global_conf) self.id = id self.gconf = global_conf end +--- Retrieve the metadata of the parameter group. This function implements the +-- *chunk* interface. +-- @return a table containing all metadata + function Param:get_info() return self.info end +--- Set the metadata of the parameter group. This function implements the +-- *chunk* interface. +-- @param info a table containing all metadata + function Param:set_info(info) self.info = info end +--- Read from the given file handle. This function implements the +-- *chunk* interface. +-- @param handle the file handle + function Param:read(handle) nerv.error_method_not_implemented() end +--- Write to the given file handle. This function implements the +-- *chunk* interface. +-- @param handle the file handle + function Param:write(handle) nerv.error_method_not_implemented() end -function Param:update(gradient) - nerv.error_method_not_implemented() -end +--- Generate zero. +-- @return zero -function Param:gen_zero() +function Param.gen_zero() return 0 end +--- The class describing a single computation node which calculates from the +-- input ports to the output ports which could be the input of others. +-- @type nerv.Layer + local Layer = nerv.class('nerv.Layer') +--- The constructor. All inheriting classes should call this base constructor to +-- initialize some predefined fields (of `self`): +-- +-- * `id`: the identifier of the layer +-- * `gconf`: a table describing the computation state and providing +-- with some global settings +-- * `lconf`: a table providing with settings dedicated for the layer. There +-- are some fields considered to be "standard" and shared by all +-- layers: +-- * `dim_in`: an array of each input port dimension (width) with order +-- * `dim_out`: an array of each output port dimension (width) with order +-- * `params`: optional, a table containing pairs of the manually bound +-- parameter name used by the layer and parameter id used to find the +-- parameter in the parameter repo +-- * `pr`: optional, the parameter repo (see `nerv.ParamRepo`) to find +-- parameters while binding, used by `nerv.Layer.find_param` +-- * `mat_type`: the type of matrix should be used when storing intermediate +-- results +-- * `loc_type`: a value from `nerv.ParamRepo.LOC_TYPES` indicating whether the +-- storage of `nerv.Param` instances is on host or device RAM +-- * `dim_in`: an array of each input port dimension (width) with order +-- * `dim_out`: an array of each output port dimension (width) with order +-- +-- @param id the identifier +-- @param global_conf see `self.gconf` +-- @param layer_conf see `self.lconf` + function Layer:__init(id, global_conf, layer_conf) self.id = id self.gconf = global_conf @@ -48,22 +106,57 @@ function Layer:__init(id, global_conf, layer_conf) self.dim_out = layer_conf.dim_out end +--- Initialize the layer, called for each epoch. + function Layer:init(batch_size) nerv.error_method_not_implemented() end -function Layer:update(bp_err, input, output) +--- Update (change the state of) the bound (tied) parameter according to the +-- calculation. +-- @param bp_err an array of row-major matrices storing the error +-- back-propagated from the output ports +-- @param input an array of row-major matrices storing the input before the +-- forward propagation +-- @param ouput an array of row-major matrices storing the output after the +-- forward propagation +-- @param t BPTT time `t` + +function Layer:update(bp_err, input, output, t) nerv.error_method_not_implemented() end -function Layer:propagate(input, output) +--- Calculate the values in output ports according to the input. +-- @param input an array of row-major matrices storing the input before the +-- forward propagation +-- @param ouput an array of row-major matrices storing the output after the +-- forward propagation +-- @param t BPTT time `t` + +function Layer:propagate(input, output, t) nerv.error_method_not_implemented() end -function Layer:back_propagate(bp_err, next_bp_err, input, output) +--- Calculate the next error value (`next_bp_err`) by back-propagation. +-- @param bp_err an array of row-major matrices storing the error +-- back-propagated from the output ports +-- @param next_bp_err an array of row-major matrices storing the next error +-- back-propagated to the input ports +-- @param input an array of row-major matrices storing the input before the +-- forward propagation +-- @param ouput an array of row-major matrices storing the output after the +-- forward propagation +-- @param t BPTT time `t` + +function Layer:back_propagate(bp_err, next_bp_err, input, output, t) nerv.error_method_not_implemented() end +--- Check the length of the dimention array. This function is recommended for +-- invocation when checking the ports in your layer implementation. +-- @param len_in the expected number of input ports (-1 if variable) +-- @param len_out the expected number of output ports (-1 if variable) + function Layer:check_dim_len(len_in, len_out) local expected_in = #self.dim_in local expected_out = #self.dim_out @@ -77,27 +170,60 @@ function Layer:check_dim_len(len_in, len_out) end end +--- Get all the parameters used by (bound to) the layer. +-- @return a `nerv.ParamRepo` instance storing all the involved parameters + function Layer:get_params() nerv.error_method_not_implemented() end +--- Lookup and bind the parameters in the repo specified in layer settings. +-- This function will lead to the invocation of `find_param` if the layer is +-- implemented appropriately. + function Layer:bind_params() nerv.error_method_not_implemented() end +--- Get two arrays describing the dimension of input and ouput. +-- @return two arrays: `<dim_in>, <dim_out>` + function Layer:get_dim() return self.dim_in, self.dim_out end +--- Set an attribute to the layer. +-- @param name the name of the attribute +-- @param value the value of the attribute + function Layer:set_attr(name, value) self[name] = value end +--- Get the contained (nested) layer inside the layer (useful for complex +-- layers like `nerv.GraphLayer`). +-- @param id the identifier of the nested layer to be find + function Layer:get_sublayer(id) nerv.error('primitive layer does not have sublayers') end -function Layer:find_param(plist, lconf, gconf, p_type, p_dim, p_gen) +--- Find the parameter according to the layer settings. This function should be +-- used as the only way to locate a parameter in the implementation of +-- `bind_params` (see `nerv.AffineLayer.bind_params`). +-- @param plist the potential names for the parameter in the order of lookup priority +-- @param lconf a table providing with settings dedicated for the layer +-- @param gconf a table describing the computation state and providing with +-- some global settings +-- @param ptype the type for constructing the parameter instance when +-- auto-generation is triggered +-- @param pdim an array specifying each dimension of the parameter when +-- auto-generation is triggered +-- @param pgen the function used to generate values when auto-generation is +-- triggered, `gconf.param_gen` will be used if `nil` + +function Layer:find_param(plist, lconf, gconf, ptype, pdim, pgen) + if type(plist) == "string" then plist = {plist} end @@ -123,14 +249,14 @@ function Layer:find_param(plist, lconf, gconf, p_type, p_dim, p_gen) pid = self.id .. '_' .. plist[1] nerv.info("param id for [%s] of layer [%s] is not found in the specified param repo, " .. "switch to auto-generate", plist_str, self.id) - local p = p_type(pid, gconf) - p.trans = self.mat_type(unpack(p_dim)) - p_gen = p_gen or gconf.param_gen - or gconf.param_random -- obsolete name - if type(p_gen) ~= "function" then + local p = ptype(pid, gconf) + p.trans = self.mat_type(unpack(pdim)) + pgen = pgen or gconf.param_gen + or gconf.param_random -- obsolete name + if type(pgen) ~= "function" then nerv.error("a param generate function is needed") end - p.trans:generate(p_gen) + p.trans:generate(pgen) return p end |