From 77b558898a2a29097d8697a59a7d23cd2a52975f Mon Sep 17 00:00:00 2001 From: Qi Liu Date: Mon, 29 Feb 2016 17:46:09 +0800 Subject: graph layer complete --- nerv/layer/graph.lua | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 nerv/layer/graph.lua (limited to 'nerv/layer/graph.lua') diff --git a/nerv/layer/graph.lua b/nerv/layer/graph.lua new file mode 100644 index 0000000..83cf810 --- /dev/null +++ b/nerv/layer/graph.lua @@ -0,0 +1,118 @@ +local GraphLayer = nerv.class('nerv.GraphLayer', 'nerv.Layer') + +function GraphLayer:__init(id, global_conf, layer_conf) + self.id = id + self.dim_in = layer_conf.dim_in + self.dim_out = layer_conf.dim_out + self.gconf = global_conf + self:graph_init(layer_conf.layer_repo, layer_conf.connections) +end + +local function parse_id(str) + local id, port, _ + _, _, id, port = string.find(str, "([a-zA-Z0-9_.]+)%[([0-9]+)%]") + if id == nil or port == nil then + _, _, id, port = string.find(str, "(.+)%[([0-9]+)%]") + if not (id == "" or id == "") then + nerv.error("wrong format of connection id") + end + end + port = tonumber(port) + return id, port +end + +local function discover(id, layers, layer_repo) + if id == '' then + id = '' + end + local ref = layers[id] + if ref == nil then + local layer = layer_repo:get_layer(id) + local dim_in, dim_out = layer:get_dim() + ref = { + layer = layer, + inputs = {}, + outputs = {}, + dim_in = dim_in, + dim_out = dim_out, + } + layers[id] = ref + end + return ref +end + +function GraphLayer:graph_init(layer_repo, connections) + self.connections = connections + self.sublayer = nerv.LayerRepo({}, nerv.ParamRepo(), self.gconf) + + -- check data dimension between connected ports + local layers = {} + layers[''] = { + inputs = {}, + outputs = {}, + dim_in = self.dim_out, + dim_out = self.dim_in, + } + for _, edge in pairs(self.connections) do + local from = edge[1] + local to = edge[2] + local id_from, port_from = parse_id(from) + local id_to, port_to = parse_id(to) + local ref_from = discover(id_from, layers, layer_repo) + local ref_to = discover(id_to, layers, layer_repo) + if ref_to.inputs[port_to] ~= nil then + nerv.error('%s has already been attached', to) + end + if ref_from.dim_out[port_from] ~= ref_to.dim_in[port_to] then + nerv.error('mismatching data dimension between %s and %s', from, to) + end + ref_from.outputs[port_from] = true + ref_to.inputs[port_to] = true + end + + -- check dangling ports + for id, ref in pairs(layers) do + if id ~= '' then + for i = 1, #ref.dim_in do + if ref.inputs[i] == nil then + nerv.error('dangling input port %d of layer %s', i, id) + end + end + for i = 1, #ref.dim_out do + if ref.outputs[i] == nil then + nerv.error('dangling output port %d os layer %s', i, id) + end + end + self.sublayer.layers[id] = ref.layer + end + end + for i = 1, #self.dim_in do + if layers[''].outputs[i] == nil then + nerv.error('dangling port %d of layer ', i) + end + end + for i = 1, #self.dim_out do + if layers[''].inputs[i] == nil then + nerv.error('dangling port %d of layer ', i) + end + end +end + +function GraphLayer:set_attr(name, value) + self[name] = value + for id, layer in pairs(self.sublayer.layers) do + layer:set_attr(name, value) + end +end + +function GraphLayer:get_sublayer(id) + return self.sublayer:get_layer(id) +end + +function GraphLayer:get_params() + local param_repos = {} + for id, layer in pairs(self.sublayer.layers) do + table.insert(param_repos, layer:get_params()) + end + return nerv.ParamRepo.merge(param_repos) +end -- cgit v1.2.3-70-g09d2 From 1a424bf9233f9b1c67ef135f1a3892b7986c5564 Mon Sep 17 00:00:00 2001 From: Qi Liu Date: Mon, 29 Feb 2016 22:05:43 +0800 Subject: add network & fix graph_layer --- nerv/Makefile | 2 +- nerv/layer/graph.lua | 46 ++++++++++++++++++++++------------- nerv/layer/rnn.lua | 4 ++-- nerv/main.lua | 4 +++- nerv/nn/init.lua | 1 + nerv/nn/network.lua | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 nerv/nn/network.lua (limited to 'nerv/layer/graph.lua') diff --git a/nerv/Makefile b/nerv/Makefile index ba97579..c9c3e42 100644 --- a/nerv/Makefile +++ b/nerv/Makefile @@ -35,7 +35,7 @@ LUA_LIBS := matrix/init.lua io/init.lua init.lua \ layer/window.lua layer/bias.lua layer/combiner.lua layer/mse.lua \ layer/elem_mul.lua layer/lstm.lua layer/lstm_gate.lua layer/dropout.lua layer/gru.lua \ layer/graph.lua layer/rnn.lua \ - nn/init.lua nn/layer_repo.lua nn/param_repo.lua nn/layer_dag.lua \ + nn/init.lua nn/layer_repo.lua nn/param_repo.lua nn/layer_dag.lua nn/network.lua \ io/sgd_buffer.lua \ tnn/init.lua tnn/sutil.lua tnn/tnn.lua diff --git a/nerv/layer/graph.lua b/nerv/layer/graph.lua index 83cf810..36a9672 100644 --- a/nerv/layer/graph.lua +++ b/nerv/layer/graph.lua @@ -21,20 +21,23 @@ local function parse_id(str) return id, port end -local function discover(id, layers, layer_repo) +function GraphLayer:discover(id, layer_repo) if id == '' then id = '' end + local layers = self.layers local ref = layers[id] if ref == nil then local layer = layer_repo:get_layer(id) local dim_in, dim_out = layer:get_dim() + self.layer_num = self.layer_num + 1 ref = { layer = layer, inputs = {}, outputs = {}, dim_in = dim_in, dim_out = dim_out, + id = self.layer_num, } layers[id] = ref end @@ -42,32 +45,37 @@ local function discover(id, layers, layer_repo) end function GraphLayer:graph_init(layer_repo, connections) - self.connections = connections - self.sublayer = nerv.LayerRepo({}, nerv.ParamRepo(), self.gconf) - - -- check data dimension between connected ports local layers = {} layers[''] = { inputs = {}, outputs = {}, dim_in = self.dim_out, dim_out = self.dim_in, + id = 0, } - for _, edge in pairs(self.connections) do - local from = edge[1] - local to = edge[2] + self.layers = layers + self.layer_num = 0 + self.connections = {} + + -- check data dimension between connected ports + for _, edge in pairs(connections) do + local from, to, time = edge[1], edge[2], edge[3] local id_from, port_from = parse_id(from) local id_to, port_to = parse_id(to) - local ref_from = discover(id_from, layers, layer_repo) - local ref_to = discover(id_to, layers, layer_repo) + local ref_from = self:discover(id_from, layer_repo) + local ref_to = self:discover(id_to, layer_repo) if ref_to.inputs[port_to] ~= nil then nerv.error('%s has already been attached', to) end if ref_from.dim_out[port_from] ~= ref_to.dim_in[port_to] then nerv.error('mismatching data dimension between %s and %s', from, to) end + if ref_from.id == 0 and ref_to.id == 0 then + nerv.error('short-circuit connection between and ') + end ref_from.outputs[port_from] = true ref_to.inputs[port_to] = true + table.insert(self.connections, {ref_from.id, port_from, ref_to.id, port_to, time}) end -- check dangling ports @@ -83,7 +91,6 @@ function GraphLayer:graph_init(layer_repo, connections) nerv.error('dangling output port %d os layer %s', i, id) end end - self.sublayer.layers[id] = ref.layer end end for i = 1, #self.dim_in do @@ -100,19 +107,26 @@ end function GraphLayer:set_attr(name, value) self[name] = value - for id, layer in pairs(self.sublayer.layers) do - layer:set_attr(name, value) + for id, ref in pairs(self.layers) do + if id ~= '' then + ref.layer:set_attr(name, value) + end end end function GraphLayer:get_sublayer(id) - return self.sublayer:get_layer(id) + if self.layers[id] == nil or id == '' then + nerv.error('layer with id %s not found', id) + end + return self.layers[id].layer end function GraphLayer:get_params() local param_repos = {} - for id, layer in pairs(self.sublayer.layers) do - table.insert(param_repos, layer:get_params()) + for id, ref in pairs(self.layers) do + if id ~= '' then + table.insert(param_repos, ref.layer:get_params()) + end end return nerv.ParamRepo.merge(param_repos) end diff --git a/nerv/layer/rnn.lua b/nerv/layer/rnn.lua index a93530f..8816891 100644 --- a/nerv/layer/rnn.lua +++ b/nerv/layer/rnn.lua @@ -29,8 +29,8 @@ function RNNLayer:__init(id, global_conf, layer_conf) local connections = { {'[1]', 'main[1]', 0}, {'main[1]', 'sigmoid[1]', 0}, - {'sigmoid[1]', 'main[2]', 0}, - {'sigmoid[1]', '[1]', 1}, + {'sigmoid[1]', 'main[2]', 1}, + {'sigmoid[1]', '[1]', 0}, } self:graph_init(layer_repo, connections) diff --git a/nerv/main.lua b/nerv/main.lua index 85e291c..0633e87 100644 --- a/nerv/main.lua +++ b/nerv/main.lua @@ -28,4 +28,6 @@ local connections = { {'output[1]', '[1]', 0}, } -local network = nerv.GraphLayer('network', global_conf, {dim_in = {20}, dim_out = {79}, layer_repo = layer_repo, connections = connections}) +local graph = nerv.GraphLayer('network', global_conf, {dim_in = {20}, dim_out = {79}, layer_repo = layer_repo, connections = connections}) + +local network = nerv.Network(graph) diff --git a/nerv/nn/init.lua b/nerv/nn/init.lua index cbaf52b..c32ea09 100644 --- a/nerv/nn/init.lua +++ b/nerv/nn/init.lua @@ -1,3 +1,4 @@ nerv.include('layer_repo.lua') nerv.include('param_repo.lua') nerv.include('layer_dag.lua') +nerv.include('network.lua') diff --git a/nerv/nn/network.lua b/nerv/nn/network.lua new file mode 100644 index 0000000..6cee08b --- /dev/null +++ b/nerv/nn/network.lua @@ -0,0 +1,68 @@ +local network = nerv.class('nerv.Network') + +function network:__init(graph) + self.layers = {} + self.socket = self:compile(graph) + for i = 1, #self.layers do + print(self.layers[i].layer.id) + local _, dim_out = self.layers[i].layer:get_dim() + for j = 1, #dim_out do + for k = 1, #self.layers[i].connections[j] do + local connections = self.layers[i].connections[j][k] + print(i, connections[1], connections[2], connections[3]) + end + end + end +end + +function network:compile(layer) + local socket = {inputs = {}, outputs = {}} + if not nerv.is_type(layer, 'nerv.GraphLayer') then + table.insert(self.layers, {layer = layer, connections = {}}) + local id = #self.layers + local dim_in, dim_out = layer:get_dim() + for i = 1, #dim_in do + socket.inputs[i] = {{id, i, 0}} + end + for i = 1, #dim_out do + socket.outputs[i] = {id, i, 0} + self.layers[id].connections[i] = {} + end + else + local sublayer_socket = {} + for id, sublayer in pairs(layer.layers) do + if id ~= '' then + sublayer_socket[sublayer.id] = self:compile(sublayer.layer) + end + end + local dim_in, _ = layer:get_dim() + for i = 1, #dim_in do + socket.inputs[i] = {} + end + for _, edge in pairs(layer.connections) do + -- id = 0 means or + local id_from, port_from = edge[1], edge[2] + local id_to, port_to = edge[3], edge[4] + local time = edge[5] + if id_from == 0 then + for _, input in pairs(sublayer_socket[id_to].inputs[port_to]) do + local id, port, t = input[1], input[2], input[3] + time + table.insert(socket.inputs[port_from], {id, port, t}) + end + else + local output = sublayer_socket[id_from].outputs[port_from] + local id, port, t = output[1], output[2], output[3] + time + if id_to == 0 then + socket.outputs[port_to] = {id, port, t} + else + local connections = self.layers[id].connections[port] + for _, input in pairs(sublayer_socket[id_to].inputs[port_to]) do + local id1, port1, t1 = input[1], input[2], input[3] + table.insert(connections, {id1, port1, t + t1}) + end + end + end + end + end + return socket +end -- cgit v1.2.3-70-g09d2 From 2ea3e139af91eb894d904d7a956e28619b1a70f6 Mon Sep 17 00:00:00 2001 From: Qi Liu Date: Tue, 1 Mar 2016 20:00:53 +0800 Subject: network init complete --- nerv/Makefile | 2 +- nerv/layer/duplicate.lua | 40 ++++++ nerv/layer/graph.lua | 3 + nerv/layer/init.lua | 1 + nerv/layer/rnn.lua | 8 +- nerv/main.lua | 18 ++- nerv/nn/network.lua | 324 +++++++++++++++++++++++++++++++++++++++++++---- 7 files changed, 367 insertions(+), 29 deletions(-) create mode 100644 nerv/layer/duplicate.lua (limited to 'nerv/layer/graph.lua') diff --git a/nerv/Makefile b/nerv/Makefile index c9c3e42..0a2aa86 100644 --- a/nerv/Makefile +++ b/nerv/Makefile @@ -34,7 +34,7 @@ LUA_LIBS := matrix/init.lua io/init.lua init.lua \ layer/init.lua layer/affine.lua layer/sigmoid.lua layer/tanh.lua layer/softmax_ce.lua layer/softmax.lua \ layer/window.lua layer/bias.lua layer/combiner.lua layer/mse.lua \ layer/elem_mul.lua layer/lstm.lua layer/lstm_gate.lua layer/dropout.lua layer/gru.lua \ - layer/graph.lua layer/rnn.lua \ + layer/graph.lua layer/rnn.lua layer/duplicate.lua\ nn/init.lua nn/layer_repo.lua nn/param_repo.lua nn/layer_dag.lua nn/network.lua \ io/sgd_buffer.lua \ tnn/init.lua tnn/sutil.lua tnn/tnn.lua diff --git a/nerv/layer/duplicate.lua b/nerv/layer/duplicate.lua new file mode 100644 index 0000000..58758e8 --- /dev/null +++ b/nerv/layer/duplicate.lua @@ -0,0 +1,40 @@ +local DuplicateLayer = nerv.class('nerv.DuplicateLayer', 'nerv.Layer') + +function DuplicateLayer:__init(id, global_conf, layer_conf) + self.id = id + self.dim_in = layer_conf.dim_in + self.dim_out = layer_conf.dim_out + self.gconf = global_conf + self:check_dim_len(1, -1) + if #self.dim_out < 1 then + nerv.error('no output specified') + end + for i = 1, #self.dim_out do + if self.dim_out[i] ~= self.dim_in[1] then + nerv.error('mismatching dimensions of outputs') + end + end +end + +function DuplicateLayer:init(batch_size) +end + +function DuplicateLayer:batch_resize(batch_size) +end + +function DuplicateLayer:update(bp_err, input, output) +end + +function DuplicateLayer:propagate(input, output) + for i = 1, #self.dim_out do + output[i]:copy_from(input[1]) + -- FIXME: use reference copy to speed up + end +end + +function DuplicateLayer:back_propagate(bp_err, next_bp_err, input, output) + next_bp_err:copy_from(bp_err[1]) + for i = 2, #self.dim_out do + next_bp_err:add(next_bp_err, bp_err[i], 1.0, 1.0) + end +end diff --git a/nerv/layer/graph.lua b/nerv/layer/graph.lua index 36a9672..d72d849 100644 --- a/nerv/layer/graph.lua +++ b/nerv/layer/graph.lua @@ -64,6 +64,9 @@ function GraphLayer:graph_init(layer_repo, connections) local id_to, port_to = parse_id(to) local ref_from = self:discover(id_from, layer_repo) local ref_to = self:discover(id_to, layer_repo) + if ref_from.outputs[port_from] ~= nil then + nerv.error('%s has already been attached', from) + end if ref_to.inputs[port_to] ~= nil then nerv.error('%s has already been attached', to) end diff --git a/nerv/layer/init.lua b/nerv/layer/init.lua index 5e3395c..6f26d4d 100644 --- a/nerv/layer/init.lua +++ b/nerv/layer/init.lua @@ -125,6 +125,7 @@ nerv.include('lstm_gate.lua') nerv.include('dropout.lua') nerv.include('gru.lua') nerv.include('rnn.lua') +nerv.include('duplicate.lua') -- The following lines are for backward compatibility, and will be removed in -- the future. The use of these names are deprecated. diff --git a/nerv/layer/rnn.lua b/nerv/layer/rnn.lua index 8816891..806ac58 100644 --- a/nerv/layer/rnn.lua +++ b/nerv/layer/rnn.lua @@ -22,6 +22,9 @@ function RNNLayer:__init(id, global_conf, layer_conf) ['nerv.SigmoidLayer'] = { sigmoid = {dim_in = {dout}, dim_out = {dout}}, }, + ['nerv.DuplicateLayer'] = { + dup = {dim_in = {dout}, dim_out = {dout, dout}}, + } } local layer_repo = nerv.LayerRepo(layers, pr, global_conf) @@ -29,8 +32,9 @@ function RNNLayer:__init(id, global_conf, layer_conf) local connections = { {'[1]', 'main[1]', 0}, {'main[1]', 'sigmoid[1]', 0}, - {'sigmoid[1]', 'main[2]', 1}, - {'sigmoid[1]', '[1]', 0}, + {'sigmoid[1]', 'dup[1]', 0}, + {'dup[1]', 'main[2]', 1}, + {'dup[2]', '[1]', 0}, } self:graph_init(layer_repo, connections) diff --git a/nerv/main.lua b/nerv/main.lua index 0633e87..5cb7d07 100644 --- a/nerv/main.lua +++ b/nerv/main.lua @@ -17,6 +17,12 @@ local layer_repo = nerv.LayerRepo( ['nerv.SigmoidLayer'] = { sigmoid = {dim_in = {23}, dim_out = {23}}, }, + ['nerv.SoftmaxLayer'] = { + softmax = {dim_in = {79}, dim_out = {79}}, + }, + ['nerv.DuplicateLayer'] = { + dup = {dim_in = {79}, dim_out = {79, 79}}, + }, }, nerv.ParamRepo(), global_conf) local connections = { @@ -24,10 +30,14 @@ local connections = { {'input[1]', 'sigmoid[1]', 0}, {'sigmoid[1]', 'rnn[1]', 0}, {'rnn[1]', 'output[1]', 0}, - {'output[1]', 'output[2]', 1}, - {'output[1]', '[1]', 0}, + {'output[1]', 'dup[1]', 0}, + {'dup[1]', 'output[2]', -1}, + {'dup[2]', 'softmax[1]', 0}, + {'softmax[1]', '[1]', 0}, } -local graph = nerv.GraphLayer('network', global_conf, {dim_in = {20}, dim_out = {79}, layer_repo = layer_repo, connections = connections}) +local graph = nerv.GraphLayer('graph', global_conf, {dim_in = {20}, dim_out = {79}, layer_repo = layer_repo, connections = connections}) + +local network = nerv.Network('network', global_conf, {network = graph}) -local network = nerv.Network(graph) +network:init(2,5) diff --git a/nerv/nn/network.lua b/nerv/nn/network.lua index 6cee08b..01290e7 100644 --- a/nerv/nn/network.lua +++ b/nerv/nn/network.lua @@ -1,15 +1,47 @@ local network = nerv.class('nerv.Network') -function network:__init(graph) +function network:__init(id, global_conf, network_conf) + self.id = id + self.dim_in = network_conf.network.dim_in + self.dim_out = network_conf.network.dim_out + self.gconf = global_conf + if self.gconf.use_cpu then + self.mat_type = self.gconf.mmat_type + else + self.mat_type = self.gconf.cumat_type + end + self.clip = network_conf.clip + self.nn_act_default = network_conf.nn_act_default + if self.nn_act_default == nil then + self.nn_act_default = 0 + end self.layers = {} - self.socket = self:compile(graph) + self.input_conn = {} + self.output_conn = {} + self.socket = self:compile(network_conf.network) + for i = 1, #self.dim_in do + local edge = self.socket.inputs[i] + local id, port, time = edge[1], edge[2], edge[3] + if self.input_conn[id][port] ~= nil then + nerv.error('duplicate edge') + end + self.input_conn[id][port] = {0, i, time} + end + for i = 1, #self.dim_out do + local edge = self.socket.outputs[i] + local id, port, time = edge[1], edge[2], edge[3] + if self.output_conn[id][port] ~= nil then + nerv.error('duplicate edge') + end + self.output_conn[id][port] = {0, i, time} + end + self.delay = 0 for i = 1, #self.layers do - print(self.layers[i].layer.id) - local _, dim_out = self.layers[i].layer:get_dim() - for j = 1, #dim_out do - for k = 1, #self.layers[i].connections[j] do - local connections = self.layers[i].connections[j][k] - print(i, connections[1], connections[2], connections[3]) + local dim_in, _ = self.layers[i]:get_dim() + for j = 1, #dim_in do + local time = self.input_conn[i][j][3] + if math.abs(time) > self.delay then + self.delay = math.abs(time) end end end @@ -18,15 +50,16 @@ end function network:compile(layer) local socket = {inputs = {}, outputs = {}} if not nerv.is_type(layer, 'nerv.GraphLayer') then - table.insert(self.layers, {layer = layer, connections = {}}) + table.insert(self.layers, layer) local id = #self.layers + self.input_conn[id] = {} + self.output_conn[id] = {} local dim_in, dim_out = layer:get_dim() for i = 1, #dim_in do - socket.inputs[i] = {{id, i, 0}} + socket.inputs[i] = {id, i, 0} end for i = 1, #dim_out do socket.outputs[i] = {id, i, 0} - self.layers[id].connections[i] = {} end else local sublayer_socket = {} @@ -35,34 +68,281 @@ function network:compile(layer) sublayer_socket[sublayer.id] = self:compile(sublayer.layer) end end - local dim_in, _ = layer:get_dim() - for i = 1, #dim_in do - socket.inputs[i] = {} - end for _, edge in pairs(layer.connections) do -- id = 0 means or local id_from, port_from = edge[1], edge[2] local id_to, port_to = edge[3], edge[4] local time = edge[5] if id_from == 0 then - for _, input in pairs(sublayer_socket[id_to].inputs[port_to]) do - local id, port, t = input[1], input[2], input[3] + time - table.insert(socket.inputs[port_from], {id, port, t}) + if socket.inputs[port_from] ~= nil then + nerv.error('duplicate input socket') end + local input = sublayer_socket[id_to].inputs[port_to] + local id, port, t = input[1], input[2], input[3] + time + socket.inputs[port_from] = {id, port, t} else local output = sublayer_socket[id_from].outputs[port_from] local id, port, t = output[1], output[2], output[3] + time if id_to == 0 then + if socket.outputs[port_to] ~= nil then + nerv.error('duplicate output socket') + end socket.outputs[port_to] = {id, port, t} else - local connections = self.layers[id].connections[port] - for _, input in pairs(sublayer_socket[id_to].inputs[port_to]) do - local id1, port1, t1 = input[1], input[2], input[3] - table.insert(connections, {id1, port1, t + t1}) + local input = sublayer_socket[id_to].inputs[port_to] + local id1, port1, t1 = input[1], input[2], input[3] + if self.input_conn[id1][port1] ~= nil or self.output_conn[id][port] ~= nil then + nerv.error('duplicate edge') end + self.input_conn[id1][port1] = {id, port, t + t1} + self.output_conn[id][port] = {id1, port1, t + t1} end end end end return socket end + +function network:init(batch_size, chunk_size) + self.batch_size = batch_size + self.chunk_size = chunk_size + + self:topsort() + + self:make_initial_store() + collectgarbage('collect') +end + +function network:topsort() + nerv.info('Network topology sort') + local degree = {} + for t = 1, self.chunk_size do + degree[t] = {} + for i = 1, #self.layers do + degree[t][i] = 0 + end + end + + for t = 1, self.chunk_size do + for i = 1, #self.layers do + local _, dim_out = self.layers[i]:get_dim() + for j = 1, #dim_out do + if self.output_conn[i][j] ~= nil then + local edge = self.output_conn[i][j] + local id, _, time = edge[1], edge[2], edge[3] + t + if time >= 1 and time <= self.chunk_size and id ~= 0 then + degree[time][id] = degree[time][id] + 1 + end + end + end + end + end + + self.queue = {} + local l = 1 + local r = 0 + for t = 1, self.chunk_size do + for i = 1, #self.layers do + if degree[t][i] == 0 then + r = r + 1 + self.queue[r] = {chunk = t, id = i} + end + end + end + while l<=r do + local t, i = self.queue[l].chunk, self.queue[l].id + l = l + 1 + local _, dim_out = self.layers[i]:get_dim() + for j = 1, #dim_out do + if self.output_conn[i][j] ~= nil then + local edge = self.output_conn[i][j] + local id, _, time = edge[1], edge[2], edge[3] + t + if time >= 1 and time <= self.chunk_size and id ~= 0 then + degree[time][id] = degree[time][id] - 1 + if degree[time][id] == 0 then + r = r + 1 + self.queue[r] = {chunk = time, id = id} + end + end + end + end + end + + if r ~= self.chunk_size * #self.layers then + nerv.error('loop detected') + end +end + +function network:make_initial_store() + nerv.info('Network initing storage') + + -- allocate memory + local memory = {} + local err_memory = {} + for t = 1 - self.delay, self.chunk_size + self.delay do + memory[t] = {} + err_memory[t] = {} + for i = 1, #self.layers do + memory[t][i] = {} + err_memory[t][i] = {} + local dim_in, dim_out = self.layers[i]:get_dim() + for j = 1, #dim_in do + err_memory[t][i][j] = self.mat_type(self.batch_size, dim_in[j]) + err_memory[t][i][j]:fill(0) + end + for j = 1, #dim_out do + memory[t][i][j] = self.mat_type(self.batch_size, dim_out[j]) + memory[t][i][j]:fill(self.nn_act_default) + end + end + -- memory[t][0] stores network input + memory[t][0] = {} + for j = 1, #self.dim_in do + memory[t][0][j] = self.mat_type(self.batch_size, self.dim_in[j]) + memory[t][0][j]:fill(self.nn_act_default) + end + -- err_memory[t][0] stores network err_input + err_memory[t][0] = {} + for j = 1, #self.dim_out do + err_memory[t][0][j] = self.mat_type(self.batch_size, self.dim_out[j]) + err_memory[t][0][j]:fill(0) + end + end + + -- connect memory and reference + self.input = {} + self.output = {} + self.err_input = {} + self.err_output = {} + for t = 1, self.chunk_size do + self.input[t] = {} + self.output[t] = {} + self.err_input[t] = {} + self.err_output[t] = {} + for i = 1, #self.layers do + self.input[t][i] = {} + self.output[t][i] = {} + self.err_input[t][i] = {} + self.err_output[t][i] = {} + local dim_in, dim_out = self.layers[i]:get_dim() + for j = 1, #dim_in do + local edge = self.input_conn[i][j] + local id, port, time = edge[1], edge[2], edge[3] + if id ~= 0 or t - time < 1 or t - time > self.chunk_size then + self.input[t][i][j] = memory[t - time][id][port] + end + if id ~= 0 then + self.err_output[t][i][j] = err_memory[t][i][j] + end + end + for j = 1, #dim_out do + local edge = self.output_conn[i][j] + local id, port, time = edge[1], edge[2], edge[3] + if id ~= 0 then + self.output[t][i][j] = memory[t][i][j] + end + if id ~= 0 or t + time < 1 or t + time > self.chunk_size then + self.err_input[t][i][j] = err_memory[t + time][id][port] + end + end + end + end + + -- check dangling reference + for t = 1, self.chunk_size do + for i = 1, #self.dim_in do + local edge = self.socket.inputs[i] + local id, port, time = edge[1], edge[2], edge[3] + if t + time >= 1 and t + time <= self.chunk_size then + if self.input[t + time][id][port] ~= nil then + nerv.error('input reference not nil') + end + self.input[t + time][id][port] = true -- just a place holder + if self.err_output[t + time][id][port] ~= nil then + nerv.error('err_output reference not nil') + end + self.err_output[t + time][id][port] = true -- just a place holder + end + end + for i = 1, #self.dim_out do + local edge = self.socket.outputs[i] + local id, port, time = edge[1], edge[2], edge[3] + if t - time >= 1 and t - time <= self.chunk_size then + if self.output[t - time][id][port] ~= nil then + nerv.error('output reference not nil') + end + self.output[t - time][id][port] = true -- just a place holder + if self.err_input[t - time][id][port] ~= nil then + nerv.error('err_output reference not nil') + end + self.err_input[t - time][id][port] = true -- just a place holder + end + end + end + for t = 1, self.chunk_size do + for i = 1, #self.layers do + local dim_in, dim_out = self.layers[i]:get_dim() + for j = 1, #dim_in do + if self.input[t][i][j] == nil then + nerv.error('input reference dangling') + end + if self.err_output[t][i][j] == nil then + nerv.error('err_output reference dangling') + end + end + for j = 1, #dim_out do + if self.output[t][i][j] == nil then + nerv.error('output reference dangling') + end + if self.err_input[t][i][j] == nil then + nerv.error('err_input reference dangling') + end + end + end + end + + -- allocate reference for legacy of previous mini-batch + self.legacy = {} + for t = 1 - self.delay, 0 do + self.legacy[t] = {} + for i = 1, #self.layers do + self.legacy[t][i] = {} + local _, dim_out = self.layers[i]:get_dim() + for j = 1, #dim_out do + self.legacy[t][i][j] = memory[t][i][j] + end + end + end +end + +function network:mini_batch_init(information) + self.info = information + self.max_chunk = 0 + for i = 1, self.batch_size do + if self.info.seq_length[i] > self.max_chunk then + self.max_chunk = self.info.seq_length[i] + end + end + for t = 1 - self.delay, 0 do + for i = 1, #self.layers do + local _, dim_out = self.layers[i]:get_dim() + for j = 1, #dim_out do + self.output[t][i][j]:copy_from(self.output[t + self.chunk_size][i][j]) + end + end + end + for t = self.max_chunk + 1, self.max_chunk + self.delay do + if t > self.chunk_size then + break + end + for i = 1, #self.layers do + local dim_in, _ = self.layers[i]:get_dim() + for j = 1, #dim_in do + self.err_output[t][i][j]:fill(0) + end + end + end +end + +function network:propagate(input, output) +end -- cgit v1.2.3-70-g09d2 From 8374e8fbc545633b6adf5c4090af8997a65778d2 Mon Sep 17 00:00:00 2001 From: Qi Liu Date: Thu, 3 Mar 2016 19:42:15 +0800 Subject: update add_prefix for graph layer --- nerv/layer/duplicate.lua | 4 ++++ nerv/layer/graph.lua | 24 ++++++++++++++++++++++++ nerv/layer/init.lua | 2 +- nerv/layer/rnn.lua | 4 ++-- nerv/main.lua | 12 +++++++++--- nerv/nn/network.lua | 15 ++++++++++++++- 6 files changed, 54 insertions(+), 7 deletions(-) (limited to 'nerv/layer/graph.lua') diff --git a/nerv/layer/duplicate.lua b/nerv/layer/duplicate.lua index 1a93b26..8988617 100644 --- a/nerv/layer/duplicate.lua +++ b/nerv/layer/duplicate.lua @@ -38,3 +38,7 @@ end function DuplicateLayer:update() end + +function DuplicateLayer:get_params() + return nerv.ParamRepo({}) +end diff --git a/nerv/layer/graph.lua b/nerv/layer/graph.lua index d72d849..1406eff 100644 --- a/nerv/layer/graph.lua +++ b/nerv/layer/graph.lua @@ -21,6 +21,30 @@ local function parse_id(str) return id, port end +function GraphLayer:add_prefix(layers, connections) + local function ap(name) + return self.id .. '.' .. name + end + + for layer_type, sublayers in pairs(layers) do + local tmp = {} + for name, layer_config in pairs(sublayers) do + tmp[ap(name)] = layer_config + end + layers[layer_type] = tmp + end + + for i = 1, #connections do + local from, to = connections[i][1], connections[i][2] + if parse_id(from) ~= '' then + connections[i][1] = ap(from) + end + if parse_id(to) ~= '' then + connections[i][2] = ap(to) + end + end +end + function GraphLayer:discover(id, layer_repo) if id == '' then id = '' diff --git a/nerv/layer/init.lua b/nerv/layer/init.lua index 39f97b1..4fabefa 100644 --- a/nerv/layer/init.lua +++ b/nerv/layer/init.lua @@ -75,7 +75,7 @@ function Layer:set_attr(name, value) end function Layer:get_sublayer(id) - nerv.error('primitive layer does not have sublayers.') + nerv.error('primitive layer does not have sublayers') end function Layer:find_param(pid_list, lconf, gconf, p_type, p_dim) diff --git a/nerv/layer/rnn.lua b/nerv/layer/rnn.lua index 806ac58..38f2326 100644 --- a/nerv/layer/rnn.lua +++ b/nerv/layer/rnn.lua @@ -27,8 +27,6 @@ function RNNLayer:__init(id, global_conf, layer_conf) } } - local layer_repo = nerv.LayerRepo(layers, pr, global_conf) - local connections = { {'[1]', 'main[1]', 0}, {'main[1]', 'sigmoid[1]', 0}, @@ -37,5 +35,7 @@ function RNNLayer:__init(id, global_conf, layer_conf) {'dup[2]', '[1]', 0}, } + self:add_prefix(layers, connections) + local layer_repo = nerv.LayerRepo(layers, pr, global_conf) self:graph_init(layer_repo, connections) end diff --git a/nerv/main.lua b/nerv/main.lua index 865aba0..7c82ebf 100644 --- a/nerv/main.lua +++ b/nerv/main.lua @@ -10,7 +10,8 @@ local global_conf = { local layer_repo = nerv.LayerRepo( { ['nerv.RNNLayer'] = { - rnn = {dim_in = {23}, dim_out = {26}}, + rnn1 = {dim_in = {23}, dim_out = {26}}, + rnn2 = {dim_in = {26}, dim_out = {26}}, }, ['nerv.AffineLayer'] = { input = {dim_in = {62}, dim_out = {23}}, @@ -30,8 +31,9 @@ local layer_repo = nerv.LayerRepo( local connections = { {'[1]', 'input[1]', 0}, {'input[1]', 'sigmoid[1]', 0}, - {'sigmoid[1]', 'rnn[1]', 0}, - {'rnn[1]', 'output[1]', 0}, + {'sigmoid[1]', 'rnn1[1]', 0}, + {'rnn1[1]', 'rnn2[1]', 0}, + {'rnn2[1]', 'output[1]', 0}, {'output[1]', 'dup[1]', 0}, {'dup[1]', 'output[2]', -1}, {'dup[2]', 'softmax[1]', 0}, @@ -65,3 +67,7 @@ for i = 1, 100 do network:back_propagate(err_input, err_output, input, output) network:update(err_input, input, output) end + +local tmp = network:get_params() + +tmp:export('../../workspace/test.param') diff --git a/nerv/nn/network.lua b/nerv/nn/network.lua index 0bbcc59..39df5f0 100644 --- a/nerv/nn/network.lua +++ b/nerv/nn/network.lua @@ -18,7 +18,8 @@ function network:__init(id, global_conf, network_conf) self.layers = {} self.input_conn = {} self.output_conn = {} - self.socket = self:compile(network_conf.network) + self.network = network_conf.network + self.socket = self:compile(self.network) for i = 1, #self.dim_in do local edge = self.socket.inputs[i] local id, port, time = edge[1], edge[2], edge[3] @@ -472,3 +473,15 @@ function network:update(bp_err, input, output) end end end + +function network:set_attr(name, value) + self.network:set_attr(name, value) +end + +function network:get_sublayer(id) + return self.network:get_sublayer(id) +end + +function network:get_params() + return self.network:get_params() +end -- cgit v1.2.3-70-g09d2