aboutsummaryrefslogtreecommitdiff
path: root/fastnn/io
diff options
context:
space:
mode:
Diffstat (limited to 'fastnn/io')
-rw-r--r--fastnn/io/Example.cpp186
-rw-r--r--fastnn/io/Example.h49
-rw-r--r--fastnn/io/example.c249
-rw-r--r--fastnn/io/example.lua38
4 files changed, 522 insertions, 0 deletions
diff --git a/fastnn/io/Example.cpp b/fastnn/io/Example.cpp
new file mode 100644
index 0000000..8f200b7
--- /dev/null
+++ b/fastnn/io/Example.cpp
@@ -0,0 +1,186 @@
+
+#include <deque>
+#include <vector>
+#include <string>
+
+
+extern "C" {
+
+#include "../threads/lib/THThread.h"
+#include "Example.h"
+#include "stdlib.h"
+#include "stdio.h"
+
+#include "../../nerv/lib/matrix/generic/elem_type.h"
+#include "common.h"
+
+extern Matrix* nerv_matrix_cuda_float_create(long nrow, long ncol, Status *status);
+void nerv_matrix_cuda_float_copy_fromd(Matrix *a, const Matrix *b,
+ int a_begin, int b_begin, int b_end,
+ Status *status);
+
+struct Example
+{
+ std::vector<Matrix *> inputs;
+ std::string id;
+ int refcount;
+};
+
+struct ExamplesRepository
+{
+ int buffer_size_;
+ THSemaphore *full_semaphore_;
+ THSemaphore *empty_semaphore_;
+ THMutex *examples_mutex_;
+
+ std::deque<Example*> examples_;
+ bool done_;
+ int refcount;
+ int gpuid;
+};
+
+Example* Example_new()
+{
+ Example *example = new Example; //(Example*)malloc(sizeof(Example));
+ example->refcount = 1;
+ return example;
+}
+
+Example* Example_newWithId(long id)
+{
+ Example* example = (Example*)(id);
+ __sync_fetch_and_add(&example->refcount, 1);
+ return example;
+}
+
+long Example_id(Example *example)
+{
+ return (long)(example);
+}
+
+void Example_destroy(Example* example)
+{
+ //printf("Example_destroy: %d\n", example->inputs.size());
+ if (NULL != example && __sync_fetch_and_add(&example->refcount, -1) == 1)
+ {
+ delete example;
+ example = NULL;
+ }
+}
+
+int Example_size(Example* example)
+{
+ return example->inputs.size();
+}
+
+Matrix* Example_at(Example* example, int idx)
+{
+ return example->inputs.at(idx);
+}
+
+void Example_pushback(Example* example, Matrix* m)
+{
+ Status status;
+ Matrix *newm = nerv_matrix_cuda_float_create(m->nrow, m->ncol, &status);
+ nerv_matrix_cuda_float_copy_fromd(newm, m, 0, 0, m->nrow, &status);
+ //__sync_fetch_and_add(&m->refcount, 1);
+ return example->inputs.push_back(newm);
+}
+
+//////////////////////////////////////////////////////////////
+
+
+ExamplesRepository* ExamplesRepository_new(int buffersize)
+{
+ ExamplesRepository *repo = new ExamplesRepository; //(ExamplesRepository*)malloc(sizeof(ExamplesRepository));
+ repo->buffer_size_ = buffersize;
+ repo->full_semaphore_ = THSemaphore_new(0);
+ repo->empty_semaphore_ = THSemaphore_new(buffersize);
+ repo->examples_mutex_ = THMutex_new();
+// repo->examples_ = new std::deque<Example*>();
+ repo->done_ = false;
+ repo->refcount = 1;
+ repo->gpuid = -1;
+ return repo;
+}
+
+ExamplesRepository* ExamplesRepository_newWithId(long id)
+{
+ ExamplesRepository *repo = (ExamplesRepository*)(id);
+ __sync_fetch_and_add(&repo->refcount, 1);
+ return repo;
+}
+
+long ExamplesRepository_id(ExamplesRepository* repo)
+{
+ return (long)(repo);
+}
+
+int ExamplesRepository_getGpuId(ExamplesRepository* repo)
+{
+ return repo->gpuid;
+}
+
+void ExamplesRepository_setGpuId(ExamplesRepository* repo, int gpuid)
+{
+ repo->gpuid = gpuid;
+}
+
+void ExamplesRepository_destroy(ExamplesRepository* repo)
+{
+ if (NULL != repo && __sync_fetch_and_add(&repo->refcount, -1) == 1)
+ {
+ if (repo->full_semaphore_)
+ THSemaphore_free(repo->full_semaphore_);
+ if (repo->empty_semaphore_)
+ THSemaphore_free(repo->empty_semaphore_);
+ if (repo->examples_mutex_)
+ THMutex_free(repo->examples_mutex_);
+ delete repo;
+ repo = NULL;
+ }
+}
+
+void AcceptExample(ExamplesRepository *repo, Example *example)
+{
+ THSemaphore_wait(repo->empty_semaphore_);
+ THMutex_lock(repo->examples_mutex_);
+ __sync_fetch_and_add(&example->refcount, 1);
+ repo->examples_.push_back(example);
+ THMutex_unlock(repo->examples_mutex_);
+ THSemaphore_signal(repo->full_semaphore_);
+}
+
+void ExamplesDone(ExamplesRepository *repo)
+{
+ for (int i = 0; i < repo->buffer_size_; i++)
+ THSemaphore_wait(repo->empty_semaphore_);
+
+ repo->done_ = true;
+ THSemaphore_signal(repo->full_semaphore_);
+}
+
+Example* ProvideExample(ExamplesRepository *repo)
+{
+ Example *ans = NULL;
+ THSemaphore_wait(repo->full_semaphore_);
+ if (repo->done_)
+ {
+ THSemaphore_signal(repo->full_semaphore_); // Increment the semaphore so
+ // the call by the next thread will not block.
+ return NULL; // no examples to return-- all finished.
+ }
+ else
+ {
+ THMutex_lock(repo->examples_mutex_);
+ ans = repo->examples_.front();
+ repo->examples_.pop_front();
+ THMutex_unlock(repo->examples_mutex_);
+ THSemaphore_signal(repo->empty_semaphore_);
+ }
+ return ans;
+}
+
+}
+
+
diff --git a/fastnn/io/Example.h b/fastnn/io/Example.h
new file mode 100644
index 0000000..1c462ab
--- /dev/null
+++ b/fastnn/io/Example.h
@@ -0,0 +1,49 @@
+#ifndef NERV_FASTNN_EXAMPLE_H
+#define NERV_FASTNN_EXAMPLE_H
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "matrix/matrix.h"
+#include "stdbool.h"
+#define STRLEN 1024
+
+
+
+typedef struct ExamplesRepository ExamplesRepository;
+typedef struct Example Example;
+
+ExamplesRepository* ExamplesRepository_new(int buffersize);
+ExamplesRepository* ExamplesRepository_newWithId(long id);
+long ExamplesRepository_id(ExamplesRepository* reop);
+void ExamplesRepository_destroy(ExamplesRepository* reop);
+void AcceptExample(ExamplesRepository *repo, Example *example);
+void ExamplesDone(ExamplesRepository *repo);
+Example* ProvideExample(ExamplesRepository *repo);
+void ExamplesRepository_setGpuId(ExamplesRepository* repo, int gpuid);
+int ExamplesRepository_getGpuId(ExamplesRepository* repo);
+
+
+Example* Example_new();
+Example* Example_newWithId(long id);
+long Example_id(Example* example);
+void Example_destroy(Example* example);
+int Example_size(Example* example);
+Matrix* Example_at(Example* example, int idx);
+void Example_pushback(Example* example, Matrix* m);
+
+
+
+
+
+
+
+
+#ifdef __cplusplus
+} // closing brace for extern "C"
+
+#endif // end Example
+
+#endif
diff --git a/fastnn/io/example.c b/fastnn/io/example.c
new file mode 100644
index 0000000..f2f0916
--- /dev/null
+++ b/fastnn/io/example.c
@@ -0,0 +1,249 @@
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <lua.h>
+#include <lualib.h>
+#include <luaT/luaT.h>
+
+
+
+#include "../threads/lib/luaTHRD.h"
+#include "Example.h"
+
+
+const char *fastnn_example_repo_tname = "fastnn.CExamplesRepo";
+const char *fastnn_example_tname = "fastnn.CExample";
+
+static int example_repo_new(lua_State *L)
+{
+ ExamplesRepository *repo = NULL ;
+ int buffersize;
+ bool shared;
+ long id;
+ if(lua_gettop(L) == 2 && lua_isboolean(L, 2))
+ {
+ shared = lua_toboolean(L, 2);
+ if (shared)
+ {
+ id = luaL_checkinteger(L, 1);
+ repo = ExamplesRepository_newWithId(id);
+ }
+ else
+ {
+ buffersize = luaL_checkinteger(L, 1);
+ repo = ExamplesRepository_new(buffersize);
+ }
+ }
+ else
+ luaL_error(L, "example: example repo new invalid arguments");
+ if (!repo)
+ luaL_error(L, "example: example new failed");
+
+ luaTHRD_pushudata(L, repo, fastnn_example_repo_tname);
+
+ return 1;
+}
+
+static int example_repo_newfromid(lua_State *L)
+{
+ ExamplesRepository *repo = NULL ;
+ if(lua_gettop(L) == 1)
+ {
+ long id = luaL_checkinteger(L, 1);
+ repo = ExamplesRepository_newWithId(id);
+ }
+ else
+ luaL_error(L, "example: example repo new invalid arguments");
+ if (!repo)
+ luaL_error(L, "example: example new failed");
+
+ luaTHRD_pushudata(L, repo, fastnn_example_repo_tname);
+
+ return 1;
+}
+
+static int example_repo_tostring(lua_State *L)
+{
+ char str[STRLEN];
+ ExamplesRepository *repo = luaTHRD_checkudata(L, 1, fastnn_example_repo_tname);
+ snprintf(str, STRLEN, "%s <%lx>", fastnn_example_repo_tname, ExamplesRepository_id(repo));
+ lua_pushstring(L, str);
+ return 1;
+}
+
+static int example_repo_id(lua_State *L)
+{
+ ExamplesRepository *repo = luaTHRD_checkudata(L, 1, fastnn_example_repo_tname);
+ lua_pushinteger(L, ExamplesRepository_id(repo));
+ return 1;
+}
+
+static int example_repo_get_gpuid(lua_State *L)
+{
+ ExamplesRepository *repo = luaTHRD_checkudata(L, 1, fastnn_example_repo_tname);
+ lua_pushinteger(L, ExamplesRepository_getGpuId(repo));
+ return 1;
+}
+
+static int example_repo_set_gpuid(lua_State *L)
+{
+ ExamplesRepository *repo = luaTHRD_checkudata(L, 1, fastnn_example_repo_tname);
+ int gpuid = luaL_checkinteger(L, 2);
+ ExamplesRepository_setGpuId(repo, gpuid);
+ return 0;
+}
+
+static int example_repo_accept(lua_State *L)
+{
+ ExamplesRepository *repo = luaTHRD_checkudata(L, 1, fastnn_example_repo_tname);
+ Example *example = luaTHRD_checkudata(L, 2, fastnn_example_tname);
+ AcceptExample(repo, example);
+ return 0;
+}
+
+static int example_repo_provide(lua_State *L)
+{
+ ExamplesRepository *repo = luaTHRD_checkudata(L, 1, fastnn_example_repo_tname);
+ Example *example = ProvideExample(repo);
+ if (NULL != example)
+ {
+ luaTHRD_pushudata(L, example, fastnn_example_tname);
+ return 1;
+ }
+ else
+ return 0;
+}
+
+static int example_repo_done(lua_State *L)
+{
+ ExamplesRepository *repo = luaTHRD_checkudata(L, 1, fastnn_example_repo_tname);
+ ExamplesDone(repo);
+ return 0;
+}
+
+static int example_repo_destroy(lua_State *L)
+{
+ ExamplesRepository *repo = luaTHRD_checkudata(L, 1, fastnn_example_repo_tname);
+ ExamplesRepository_destroy(repo);
+ return 0;
+}
+
+/******************************************/
+
+static int example_new(lua_State *L)
+{
+
+ Example *example = NULL;
+ if(lua_gettop(L) == 0)
+ {
+ example = Example_new();
+ }
+ else if(lua_gettop(L) == 1)
+ {
+ long id = luaL_checkinteger(L, 1);
+ example = Example_newWithId(id);
+ }
+ else
+ luaL_error(L, "example: example new invalid arguments");
+ if (!example)
+ luaL_error(L, "example: example failed");
+
+ luaTHRD_pushudata(L, example, fastnn_example_tname);
+
+ return 1;
+}
+
+static int example_id(lua_State *L)
+{
+ Example *example = luaTHRD_checkudata(L, 1, fastnn_example_tname);
+ lua_pushinteger(L, Example_id(example));
+ return 1;
+}
+
+static int example_tostring(lua_State *L)
+{
+ char str[STRLEN];
+ Example* example = luaTHRD_checkudata(L, 1, fastnn_example_tname);
+ snprintf(str, STRLEN, "%s <%lx>", fastnn_example_tname, Example_id(example));
+ lua_pushstring(L, str);
+ return 1;
+}
+
+static int example_size(lua_State *L)
+{
+ Example *example = luaTHRD_checkudata(L, 1, fastnn_example_tname);
+ lua_pushinteger(L, Example_size(example));
+ return 1;
+}
+
+static int example_at(lua_State *L)
+{
+ Example *example = luaTHRD_checkudata(L, 1, fastnn_example_tname);
+ int idx = luaL_checkinteger(L, 2);
+ Matrix *m = Example_at(example, idx);
+ luaTHRD_pushudata(L, m, "nerv.CuMatrixFloat");
+ //luaT_pushudata(L, m, "nerv.CuMatrixFloat");
+ return 1;
+}
+
+static int example_pushback(lua_State *L)
+{
+ Example *example = luaTHRD_checkudata(L, 1, fastnn_example_tname);
+ Matrix *m = luaTHRD_checkudata(L, 2, "nerv.CuMatrixFloat");
+ //Matrix *m = luaT_checkudata(L, 2, "nerv.CuMatrixFloat");
+ Example_pushback(example, m);
+ return 0;
+}
+
+static int example_destroy(lua_State *L)
+{
+ Example *example = luaTHRD_checkudata(L, 1, fastnn_example_tname);
+ Example_destroy(example);
+ //printf("example_destroy ... end\n");
+ return 0;
+}
+
+
+static const struct luaL_Reg example_repo__ [] = {
+ {"new", example_repo_new},
+ {"newfromid", example_repo_newfromid},
+ {"__tostring", example_repo_tostring},
+ {"id", example_repo_id},
+ {"get_gpuid", example_repo_get_gpuid},
+ {"set_gpuid", example_repo_set_gpuid},
+ {"accept", example_repo_accept},
+ {"provide", example_repo_provide},
+ {"done", example_repo_done},
+ {"free", example_repo_destroy},
+ {NULL, NULL}
+};
+
+static const struct luaL_Reg example__ [] = {
+ {"new", example_new},
+ {"__tostring", example_tostring},
+ {"id", example_id},
+ {"size", example_size},
+ {"at", example_at},
+ {"pushback", example_pushback},
+ {"free", example_destroy},
+ {NULL, NULL}
+};
+
+void fastnn_init_example(lua_State *L)
+{
+ luaT_newmetatable(L, fastnn_example_tname, NULL, example_new, example_destroy, NULL);
+ luaL_register(L, NULL, example__);
+ lua_pop(L, 1);
+}
+
+void fastnn_init_example_repo(lua_State *L)
+{
+ luaT_newmetatable(L, fastnn_example_repo_tname, NULL, example_repo_new, example_repo_destroy, NULL);
+ luaL_register(L, NULL, example_repo__);
+ lua_pop(L, 1);
+}
+
+
+
+
diff --git a/fastnn/io/example.lua b/fastnn/io/example.lua
new file mode 100644
index 0000000..8808791
--- /dev/null
+++ b/fastnn/io/example.lua
@@ -0,0 +1,38 @@
+
+local T = require 'libthreads'
+local C = require 'libfastnn'
+
+local ExamplesRepo = nerv.class("fastnn.ExamplesRepo")
+local Example = nerv.class("fastnn.Example")
+
+fastnn.CExamplesRepo = C.CExamplesRepo
+fastnn.CExample = C.CExample
+
+function ExamplesRepo:__init(shareid)
+ if nil ~= shareid then
+ -- print(shareid)
+ self.repo = C.CExamplesRepo(shareid, true)
+ -- print(self.repo:__tostring())
+ end
+end
+
+
+function Example:PrepareData(data, global_transf, trains_id)
+ local example = fastnn.CExample()
+ if nil ~= data then
+ for id, cm in pairs(data) do
+ if trains_id[id] and nil ~= global_transf then
+ local tcm = nerv.speech_utils.normalize(cm, global_transf)
+ example:pushback(tcm)
+ else
+ example:pushback(cm)
+ end
+
+ end
+ end
+
+ return example
+end
+
+
+
Powered by cgit, Themed by Ted Yin.