diff options
| -rw-r--r-- | nerv/layer/duplicate.lua | 12 | ||||
| -rw-r--r-- | nerv/layer/identity.lua | 33 | ||||
| -rw-r--r-- | nerv/nn/network.lua | 18 |
3 files changed, 53 insertions, 10 deletions
diff --git a/nerv/layer/duplicate.lua b/nerv/layer/duplicate.lua index 58758e8..fbd4a9e 100644 --- a/nerv/layer/duplicate.lua +++ b/nerv/layer/duplicate.lua @@ -16,13 +16,10 @@ function DuplicateLayer:__init(id, global_conf, layer_conf) end end -function DuplicateLayer:init(batch_size) +function DuplicateLayer:init() end -function DuplicateLayer:batch_resize(batch_size) -end - -function DuplicateLayer:update(bp_err, input, output) +function DuplicateLayer:batch_resize() end function DuplicateLayer:propagate(input, output) @@ -32,9 +29,12 @@ function DuplicateLayer:propagate(input, output) end end -function DuplicateLayer:back_propagate(bp_err, next_bp_err, input, output) +function DuplicateLayer:back_propagate(bp_err, next_bp_err) next_bp_err:copy_from(bp_err[1]) for i = 2, #self.dim_out do next_bp_err:add(next_bp_err, bp_err[i], 1.0, 1.0) end end + +function DuplicateLayer:update() +end diff --git a/nerv/layer/identity.lua b/nerv/layer/identity.lua new file mode 100644 index 0000000..dc796fb --- /dev/null +++ b/nerv/layer/identity.lua @@ -0,0 +1,33 @@ +local IdentityLayer = nerv.class('nerv.IdentityLayer', 'nerv.Layer') + +function IdentityLayer:__init(id, global_conf, layer_conf) + self.id = id + self.dim_in = layer_conf.dim_in + self.dim_out = layer_conf.dim_out + self.gconf = global_conf + self:check_dim_len(1, 1) + if self.dim_in[1] ~= self.dim_out[1] then + nerv.error('mismatching dimensions of input and output') + end +end + +function IdentityLayer:init() +end + +function IdentityLayer:batch_resize() +end + +function IdentityLayer:propagate(input, output) + output[1]:copy_from(input[1]) +end + +function IdentityLayer:back_propagate(bp_err, next_bp_err) + next_bp_err[1]:copy_from(bp_err) +end + +function IdentityLayer:update() +end + +function IdentityLayer:get_params() + return nerv.ParamRepo({}) +end diff --git a/nerv/nn/network.lua b/nerv/nn/network.lua index e1a9629..3cf052b 100644 --- a/nerv/nn/network.lua +++ b/nerv/nn/network.lua @@ -118,7 +118,7 @@ function network:init(batch_size, chunk_size) end function network:topsort() - nerv.info('Network topology sort') + nerv.info('network topology sort') local degree = {} for t = 1, self.chunk_size do degree[t] = {} @@ -133,7 +133,7 @@ function network:topsort() for j = 1, #dim_out do if self.output_conn[i][j] ~= nil then local edge = self.output_conn[i][j] - local id, _, time = edge[1], edge[2], edge[3] + t + local id, time = edge[1], edge[3] + t if time >= 1 and time <= self.chunk_size and id ~= 0 then degree[time][id] = degree[time][id] + 1 end @@ -160,7 +160,7 @@ function network:topsort() for j = 1, #dim_out do if self.output_conn[i][j] ~= nil then local edge = self.output_conn[i][j] - local id, _, time = edge[1], edge[2], edge[3] + t + local id, time = edge[1], edge[3] + t if time >= 1 and time <= self.chunk_size and id ~= 0 then degree[time][id] = degree[time][id] - 1 if degree[time][id] == 0 then @@ -178,7 +178,7 @@ function network:topsort() end function network:make_initial_store() - nerv.info('Network initing storage') + nerv.info('network initing storage') -- allocate memory local memory = {} @@ -386,6 +386,7 @@ function network:mini_batch_init(information) table.insert(self.border[chunk], i) end end + -- copy legacy for t = 1 - self.delay, 0 do for i = 1, #self.layers do local _, dim_out = self.layers[i]:get_dim() @@ -398,6 +399,7 @@ function network:mini_batch_init(information) end end end + -- flush border gradient for t = self.max_length + 1, self.max_length + self.delay do if t > self.chunk_size then break @@ -419,6 +421,7 @@ function network:propagate(input, output) if t <= self.max_length then self.layers[id]:propagate(self.input[t][id], self.output[t][id], t) end + -- flush border activation for j = 1, #self.border[t] do local batch = self.border[t][j] local _, dim_out = self.layers[id]:get_dim() @@ -437,6 +440,7 @@ function network:back_propagate(bp_err, next_bp_err, input, output) for i = #self.queue, 1, -1 do local t, id = self.queue[i].chunk, self.queue[i].id if t <= self.max_length then + -- flush border gradient for j = 1, #self.border[t] do local batch = self.border[t][j] local dim_in, _ = self.layers[id]:get_dim() @@ -445,6 +449,12 @@ function network:back_propagate(bp_err, next_bp_err, input, output) end end self.layers[id]:back_propagate(self.err_input[t][id], self.err_output[t][id], self.input[t][id], self.output[t][id], t) + if self.clip ~= nil then + local dim_in, _ = self.layers[id]:get_dim() + for j = 1, #dim_in do + self.err_output[t][id][j]:clip(-self.clip, self.clip) + end + end end end end |