fix bug in affine.lua

2 files changed, 13 insertions, 12 deletions

diff --git a/nerv/layer/affine.lua b/nerv/layer/affine.lua
index 8b4751c..1ac4681 100644
--- a/nerv/layer/affine.lua
+++ b/nerv/layer/affine.lua
@@ -88,7 +88,7 @@ local AffineLayer = nerv.class('nerv.AffineLayer', 'nerv.Layer')
 -- @param layer_conf a table providing with settings dedicated for the layer,
 -- for `layer_conf` fields that are shared by all layers, see
 -- `nerv.Layer.__init`. This fields can be specified:
--- * `activation`: the type of the activation function layer, also known as \sigma in \sigma(Wx + b). Default value none (no activation function).
+-- * `activation`: the type of the activation function layer, also known as \sigma in \sigma(Wx + b). The activation function layer must gurantee not use parameter `input` in its `back_propagate` function. Default value none (no activation function).
 -- * `no_bias`: a bool value indicates use bias parameter or not. Default value false.
 -- * `param_type`: a string table has the same length with `dim_in`, indicates the parameter type for every input. 'D' for diagonal weight matrix, 'N' for normal weight matrix. Default 'N' for every input.
 -- The affine layer requires parameters to be bound, the
@@ -99,11 +99,12 @@ local AffineLayer = nerv.class('nerv.AffineLayer', 'nerv.Layer')
 
 function AffineLayer:__init(id, global_conf, layer_conf)
     nerv.Layer.__init(self, id, global_conf, layer_conf)
-    self.param_type = layer_conf.param_type or table.vector(#self.dim_in, 'N')
     self:check_dim_len(-1, 1) -- exactly one output, allow multiple inputs
+    self.param_type = layer_conf.param_type or table.vector(#self.dim_in, 'N')
     if layer_conf.activation then
         self.activation = layer_conf.activation('', global_conf, {dim_in = {self.dim_out[1]}, dim_out = {self.dim_out[1]}})
     end
+    self.no_bias = layer_conf.no_bias
     self:bind_params()
 end
 
@@ -138,7 +139,7 @@ function AffineLayer:bind_params()
 end
 
 function AffineLayer:init(batch_size)
-    if self.dim_out[1] ~= self.bp.trans:ncol() then
+    if not self.no_bias and self.dim_out[1] ~= self.bp.trans:ncol() then
         nerv.error("mismatching dimensions of linear transform and bias paramter")
     end
     for i = 1, #self.dim_in do
@@ -154,10 +155,8 @@ function AffineLayer:init(batch_size)
         self.bp:train_init()
     end
     if self.activation then
-        self.act_bak = self.mat_type(batch_size, self.dim_out[1])
-        self.act_bak:fill(0)
-        self.err_bak = self.mat_type(batch_size, self.dim_out[1])
-        self.err_bak:fill(0)
+        self.bak_mat = self.mat_type(batch_size, self.dim_out[1])
+        self.bak_mat:fill(0)
     end
 end
 
@@ -168,6 +167,9 @@ end
 function AffineLayer:update()
     for i = 1, #self.dim_in do
         self["ltp" .. i]:update_by_err_input()
+        if self.param_type[i] == 'D' then
+            self['ltp' .. i].trans:diagonalize()
+        end
     end
     if not self.no_bias then
         self.bp:update_by_gradient()
@@ -175,7 +177,7 @@ function AffineLayer:update()
 end
 
 function AffineLayer:propagate(input, output)
-    local result = self.activation and self.act_bak or output[1]
+    local result = self.activation and self.bak_mat or output[1]
     -- apply linear transform
     result:mul(input[1], self.ltp1.trans, 1.0, 0.0, 'N', 'N')
     for i = 2, #self.dim_in do
@@ -186,15 +188,15 @@ function AffineLayer:propagate(input, output)
         result:add_row(self.bp.trans, 1.0)
     end
     if self.activation then
-        self.activation:propagate({self.act_bak}, output)
+        self.activation:propagate({result}, output)
     end
 end
 
 function AffineLayer:back_propagate(bp_err, next_bp_err, input, output)
+    local result = self.activation and self.bak_mat or bp_err[1]
     if self.activation then
-        self.activation:back_propagate(bp_err, {self.err_bak}, {self.act_bak}, output)
+        self.activation:back_propagate(bp_err, {result}, {result}, output)
     end
-    local result = self.activation and self.err_bak or bp_err[1]
     for i = 1, #self.dim_in do
         next_bp_err[i]:mul(result, self["ltp" .. i].trans, 1.0, 0.0, 'N', 'T')
         self["ltp" .. i]:back_propagate_by_err_input(result, input[i])
diff --git a/nerv/nn/network.lua b/nerv/nn/network.lua
index bf69ccc..d0d5462 100644
--- a/nerv/nn/network.lua
+++ b/nerv/nn/network.lua
@@ -416,7 +416,6 @@ function network:make_initial_store()
             local dim_in, dim_out = self.layers[i]:get_dim()
             for j = 1, #dim_in do
                 if self.input[t][i][j] == nil then
-                    print(t,i,j,self.layers[i].id)
                     nerv.error('input reference dangling')
                 end
                 if self.err_output[t][i][j] == nil then