diff options
Diffstat (limited to 'nerv/layer/dropout.lua')
-rw-r--r-- | nerv/layer/dropout.lua | 77 |
1 files changed, 77 insertions, 0 deletions
diff --git a/nerv/layer/dropout.lua b/nerv/layer/dropout.lua new file mode 100644 index 0000000..42660cc --- /dev/null +++ b/nerv/layer/dropout.lua @@ -0,0 +1,77 @@ +local DropoutLayer = nerv.class("nerv.DropoutLayer", "nerv.Layer") + +function DropoutLayer:__init(id, global_conf, layer_conf) + self.id = id + self.gconf = global_conf + if self.gconf.use_cpu then + self.mat_type = self.gconf.mmat_type + else + self.mat_type = self.gconf.cumat_type + end + self.rate = layer_conf.dropout_rate or global_conf.dropout_rate + if self.rate == nil then + nerv.warning("[DropoutLayer:propagate] dropout rate is not set") + end + self.dim_in = layer_conf.dim_in + self.dim_out = layer_conf.dim_out + self:check_dim_len(1, 1) -- two inputs: nn output and label +end + +function DropoutLayer:init(batch_size, chunk_size) + if self.dim_in[1] ~= self.dim_out[1] then + nerv.error("mismatching dimensions of input and output") + end + if chunk_size == nil then + chunk_size = 1 + end + self.mask = {} + for t = 1, chunk_size do + self.mask[t] = self.mat_type(batch_size, self.dim_in[1]) + end +end + +function DropoutLayer:batch_resize(batch_size, chunk_size) + if chunk_size == nil then + chunk_size = 1 + end + for t = 1, chunk_size do + if self.mask[t] == nil or self.mask[t]:nrow() ~= batch_size then + self.mask[t] = self.mat_type(batch_size, self.dim_in[1]) + end + end +end + +function DropoutLayer:propagate(input, output, t) + if t == nil then + t = 1 + end + if self.rate then + self.mask[t]:rand_uniform() + -- since we will lose a portion of the actvations, we multiply the + -- activations by 1 / (1 - rate) to compensate + self.mask[t]:thres_mask(self.mask[t], self.rate, + 0, 1 / (1.0 - self.rate)) + output[1]:mul_elem(input[1], self.mask[t]) + else + output[1]:copy_fromd(input[1]) + end +end + +function DropoutLayer:update(bp_err, input, output, t) + -- no params, therefore do nothing +end + +function DropoutLayer:back_propagate(bp_err, next_bp_err, input, output, t) + if t == nil then + t = 1 + end + if self.rate then + next_bp_err[1]:mul_elem(bp_err[1], self.mask[t]) + else + next_bp_err[1]:copy_fromd(bp_err[1]) + end +end + +function DropoutLayer:get_params() + return nerv.ParamRepo({}) +end |