diff options
Diffstat (limited to 'nerv/nn/network.lua')
| -rw-r--r-- | nerv/nn/network.lua | 18 |
1 files changed, 14 insertions, 4 deletions
diff --git a/nerv/nn/network.lua b/nerv/nn/network.lua index e1a9629..3cf052b 100644 --- a/nerv/nn/network.lua +++ b/nerv/nn/network.lua @@ -118,7 +118,7 @@ function network:init(batch_size, chunk_size) end function network:topsort() - nerv.info('Network topology sort') + nerv.info('network topology sort') local degree = {} for t = 1, self.chunk_size do degree[t] = {} @@ -133,7 +133,7 @@ function network:topsort() for j = 1, #dim_out do if self.output_conn[i][j] ~= nil then local edge = self.output_conn[i][j] - local id, _, time = edge[1], edge[2], edge[3] + t + local id, time = edge[1], edge[3] + t if time >= 1 and time <= self.chunk_size and id ~= 0 then degree[time][id] = degree[time][id] + 1 end @@ -160,7 +160,7 @@ function network:topsort() for j = 1, #dim_out do if self.output_conn[i][j] ~= nil then local edge = self.output_conn[i][j] - local id, _, time = edge[1], edge[2], edge[3] + t + local id, time = edge[1], edge[3] + t if time >= 1 and time <= self.chunk_size and id ~= 0 then degree[time][id] = degree[time][id] - 1 if degree[time][id] == 0 then @@ -178,7 +178,7 @@ function network:topsort() end function network:make_initial_store() - nerv.info('Network initing storage') + nerv.info('network initing storage') -- allocate memory local memory = {} @@ -386,6 +386,7 @@ function network:mini_batch_init(information) table.insert(self.border[chunk], i) end end + -- copy legacy for t = 1 - self.delay, 0 do for i = 1, #self.layers do local _, dim_out = self.layers[i]:get_dim() @@ -398,6 +399,7 @@ function network:mini_batch_init(information) end end end + -- flush border gradient for t = self.max_length + 1, self.max_length + self.delay do if t > self.chunk_size then break @@ -419,6 +421,7 @@ function network:propagate(input, output) if t <= self.max_length then self.layers[id]:propagate(self.input[t][id], self.output[t][id], t) end + -- flush border activation for j = 1, #self.border[t] do local batch = self.border[t][j] local _, dim_out = self.layers[id]:get_dim() @@ -437,6 +440,7 @@ function network:back_propagate(bp_err, next_bp_err, input, output) for i = #self.queue, 1, -1 do local t, id = self.queue[i].chunk, self.queue[i].id if t <= self.max_length then + -- flush border gradient for j = 1, #self.border[t] do local batch = self.border[t][j] local dim_in, _ = self.layers[id]:get_dim() @@ -445,6 +449,12 @@ function network:back_propagate(bp_err, next_bp_err, input, output) end end self.layers[id]:back_propagate(self.err_input[t][id], self.err_output[t][id], self.input[t][id], self.output[t][id], t) + if self.clip ~= nil then + local dim_in, _ = self.layers[id]:get_dim() + for j = 1, #dim_in do + self.err_output[t][id][j]:clip(-self.clip, self.clip) + end + end end end end |