add TNetReader

5 files changed, 118 insertions, 3 deletions

diff --git a/Makefile b/Makefile
index 531e43f..78c18da 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,9 @@
 .PHONY: tnet
 INCLUDE := -I $(BUILD_DIR)/luajit-2.0/include/luajit-2.0
-all: tnet $(OBJ_DIR)/tnet_io/test $(LIB_DIR)/libspeech.so
+all: tnet $(OBJ_DIR)/tnet_io/test $(LIB_DIR)/libspeech.so $(LUA_DIR)/speech/init.lua
+$(LUA_DIR)/speech/init.lua: init.lua
+	mkdir -p $(LUA_DIR)/speech
+	cp $< $@
 $(LIB_DIR)/libspeech.so: $(OBJ_DIR)/tnet_io/cwrapper.o $(OBJ_DIR)/init.o $(OBJ_DIR)/tnet_io/init.o $(OBJ_DIR)/tnet_io/libKaldiLib.a
 	gcc -shared -o $@ $(OBJ_DIR)/tnet_io/cwrapper.o $(OBJ_DIR)/init.o $(OBJ_DIR)/tnet_io/libKaldiLib.a $(OBJ_DIR)/tnet_io/init.o -lstdc++ -Wl,-rpath=$(LIB_DIR) -L$(LIB_DIR) -lnerv
 $(OBJ_DIR)/tnet_io/test: $(OBJ_DIR)/tnet_io/cwrapper.o $(OBJ_DIR)/tnet_io/test.o $(OBJ_DIR)/tnet_io/libKaldiLib.a
diff --git a/examples/tnet_preprocessing_example2.lua b/examples/tnet_preprocessing_example2.lua
new file mode 100644
index 0000000..ae8d86b
--- /dev/null
+++ b/examples/tnet_preprocessing_example2.lua
@@ -0,0 +1,68 @@
+require 'speech.init'
+gconf = {mat_type = nerv.CuMatrixFloat,
+        batch_size = 158}
+param_repo = nerv.ParamRepo({"global_transf.nerv"})
+
+sublayer_repo = nerv.LayerRepo(
+    {
+        ["nerv.BiasLayer"] =
+        {
+            blayer1 = {{bias = "bias1"}, {dim_in = {429}, dim_out = {429}}},
+            blayer2 = {{bias = "bias2"}, {dim_in = {429}, dim_out = {429}}}
+        },
+        ["nerv.WindowLayer"] =
+        {
+            wlayer1 = {{window = "window1"}, {dim_in = {429}, dim_out = {429}}},
+            wlayer2 = {{window = "window2"}, {dim_in = {429}, dim_out = {429}}}
+        }
+    }, param_repo, gconf)
+
+layer_repo = nerv.LayerRepo(
+    {
+        ["nerv.DAGLayer"] =
+        {
+            main = {{}, {
+                dim_in = {429}, dim_out = {429},
+                sub_layers = sublayer_repo,
+                connections = {
+                    ["<input>[1]"] = "blayer1[1]",
+                    ["blayer1[1]"] = "wlayer1[1]",
+                    ["wlayer1[1]"] = "blayer2[1]",
+                    ["blayer2[1]"] = "wlayer2[1]",
+                    ["wlayer2[1]"] = "<output>[1]"
+                }
+            }}
+        }
+    }, param_repo, gconf)
+
+reader = nerv.TNetReader({},
+    {
+        id = "main_scp",
+        scp_file = "/slfs1/users/mfy43/swb_ivec/train_bp.scp",
+        conf_file = "/slfs1/users/mfy43/swb_ivec/plp_0_d_a.conf",
+        frm_ext = 5,
+        mlfs = {
+            ref = {
+                file = "/slfs1/users/mfy43/swb_ivec/ref.mlf",
+                format = "map",
+                format_arg = "/slfs1/users/mfy43/swb_ivec/dict",
+                dir = "*/",
+                ext = "lab"
+            }
+        },
+        global_transf = layer_repo:get_layer("main")
+    })
+
+utter = reader:get_data()
+-- print(utter.main_scp)
+print(utter.ref)
+-- cf2 = nerv.ChunkFile("feat_256", "r")
+-- input = cf2:read_chunk("input", gconf)
+
+-- for i = 0, 157 - 10 do
+--     row_diff = input.trans[i] - utter.main_scp[i]
+--     for j = 0, row_diff:ncol() - 1 do
+--         nerv.utils.printf("%.8f ", row_diff[j])
+--     end
+--     nerv.utils.printf("\n")
+-- end
diff --git a/init.lua b/init.lua
new file mode 100644
index 0000000..1f20096
--- /dev/null
+++ b/init.lua
@@ -0,0 +1,43 @@
+require 'libspeech'
+local TNetReader = nerv.class("nerv.TNetReader", "nerv.DataReader")
+
+function TNetReader:__init(global_conf, reader_conf)
+    self.feat_id = reader_conf.id
+    self.frm_ext = reader_conf.frm_ext
+    self.feat_repo = nerv.TNetFeatureRepo(reader_conf.scp_file,
+                                          reader_conf.conf_file,
+                                          reader_conf.frm_ext)
+    self.lab_repo = {}
+    for id, mlf_spec in pairs(reader_conf.mlfs) do
+        self.lab_repo[id] = nerv.TNetLabelRepo(mlf_spec.file,
+                                            mlf_spec.format,
+                                            mlf_spec.format_arg,
+                                            mlf_spec.dir,
+                                            mlf_spec.ext)
+    end
+    self.global_transf = reader_conf.global_transf
+end
+
+function TNetReader:get_data()
+    local res = {}
+    local frm_ext = self.frm_ext
+    local step = frm_ext * 2 + 1
+    local feat_utter = self.feat_repo:cur_utter()
+    local expanded = nerv.CuMatrixFloat(feat_utter:nrow(), feat_utter:ncol() * step)
+    expanded:expand_frm(nerv.CuMatrixFloat.new_from_host(feat_utter), frm_ext)
+    local rearranged = expanded:create()
+    rearranged:rearrange_frm(expanded, step)
+    local input = {rearranged}
+    local output = {rearranged:create()}
+    self.global_transf:init()
+    self.global_transf:propagate(input, output)
+    expanded = nerv.CuMatrixFloat(output[1]:nrow() - frm_ext * 2, output[1]:ncol())
+    expanded:copy_fromd(output[1], frm_ext, feat_utter:nrow() - frm_ext)
+    res[self.feat_id] = expanded
+    for id, repo in pairs(self.lab_repo) do
+        local lab_utter = repo:get_utter(self.feat_repo, expanded:nrow())
+        res[id] = lab_utter
+    end
+    self.feat_repo:next()
+    return res
+end
diff --git a/tnet_io/cwrapper.h b/tnet_io/cwrapper.h
index 810901b..a34f090 100644
--- a/tnet_io/cwrapper.h
+++ b/tnet_io/cwrapper.h
@@ -1,6 +1,7 @@
 #ifndef NERV_TNET_IO_CWRAPPER
 #define NERV_TNET_IO_CWRAPPER
 #include "../../matrix/generic/matrix.h"
+#include "../../common.h"
 #ifdef __cplusplus
 extern "C" {
 #endif
diff --git a/tnet_io/test.c b/tnet_io/test.c
index a5b8f55..8c06805 100644
--- a/tnet_io/test.c
+++ b/tnet_io/test.c
@@ -22,7 +22,7 @@ int main() {
             "/slfs1/users/mfy43/swb_ivec/train_bp.scp",
             "/slfs1/users/mfy43/swb_ivec/plp_0_d_a.conf", 5);
     Matrix *feat_utter;
-    feat_utter = tnet_feature_repo_read_utterance(feat_repo);
+    feat_utter = tnet_feature_repo_read_utterance(feat_repo, NULL);
 
     TNetLabelRepo *lab_repo = tnet_label_repo_new(
             "/slfs1/users/mfy43/swb_ivec/ref.mlf",
@@ -33,7 +33,7 @@ int main() {
     Matrix *lab_utter = tnet_label_repo_read_utterance(lab_repo,
             feat_utter->nrow - 5 * 2,
             tnet_feature_repo_current_samplerate(feat_repo),
-            tnet_feature_repo_current_tag(feat_repo));
+            tnet_feature_repo_current_tag(feat_repo), NULL);
     print_nerv_matrix(lab_utter);
     return 0;
 }