diff options
45 files changed, 129691 insertions, 65 deletions
@@ -3,3 +3,4 @@ install/ build/ *.swp *.swo +speech/ @@ -7,7 +7,7 @@ luajit: luarocks: PREFIX=$(PREFIX) ./tools/build_luarocks.sh install: - cd nerv; $(PREFIX)/bin/luarocks make + cd nerv; $(PREFIX)/bin/luarocks make CFLAGS=$(CFLAGS) speech: cd speech/speech_utils; $(PREFIX)/bin/luarocks make cd speech/htk_io; $(PREFIX)/bin/luarocks make @@ -1,24 +1,32 @@ #The Nerv Toolkit User Manual# -NOTE: This readme is obsolete and will be rearranged, for further information, please check http://determinant.github.io/nerv/ +NOTE: This readme is obsolete and will be rearranged, for further information, please check http://nerv-sjtu.github.io/nerv/ This user manual will information about how to use __Nerv__ and __Nerv__'s interface. -##How to use## +##How to make and start using## First make sure you have __lua__ and __CUDA__ installed on your computer. __Nerv__ is currently developed via github.You can download and make __Nerv__ by doing the following: ``` cd ~ git clone https://github.com/Nerv-SJTU/nerv.git cd nerv -git clone https://github.com/Nerv-SJTU/nerv-speech.git speech git submodule init && git submodule update make +#To include some new CUDA feature(e.x. atomicCAS), use "make CFLAGS=-D__NERV_FUTURE_CUDA_7" + +#further, if you want the speech modules +git clone https://github.com/Nerv-SJTU/nerv-speech.git speech make speech ``` The `git submodule` command is for the __luajit__ repository inside __Nerv__. Now, you can try to run some example scripts. ``` -./nerv examples/cumatrix_example.lua +./install/bin/nerv examples/cumatrix_example.lua +``` +To get an example of DNN(for ASR) training, run(this requires the speech modules) +You need to be at or (copy files from) `/slfs1`(SJTU speechlab cluster) to get this running. +``` +./install/bin/nerv nerv/examples/asr_trainer.lua nerv/examples/swb_baseline.lua ``` ##How to contribute## @@ -28,19 +36,19 @@ The pull&merge request can be found on your dashboard in github. See this [sync- ##Nerv Packages## * __luaT__ Nerv uses [luaT]\(a [Torch] library\) to define lua class in C. -* __[The Nerv OOP](doc/nerv_class.md)__ +* __[The Nerv OOP](nerv/doc/nerv_class.md)__ Enables object-oriented programming in Nerv. -* __[The Nerv utility functions](doc/nerv.md)__ +* __[The Nerv utility functions](nerv/doc/nerv.md)__ Inlcudes some utility functions from luaT to implement __Nerv.Class__. -* __[The Nerv Matrix Package](doc/nerv_matrix.md)__ +* __[The Nerv Matrix Package](nerv/doc/nerv_matrix.md)__ The matrix package is a basic package in __Nerv__ that is used to store and manipulate matrices. -* __[The Nerv IO Package](doc/nerv_io.md)__ +* __[The Nerv IO Package](nerv/doc/nerv_io.md)__ The IO package is used to read and write parameters to file. -* __[The Nerv Parameter Package](doc/nerv_param.md)__ +* __[The Nerv Parameter Package](nerv/doc/nerv_param.md)__ The parameter package is used to store, read model parameters from file. -* __[The Nerv Layer Package](doc/nerv_layer.md)__ +* __[The Nerv Layer Package](nerv/doc/nerv_layer.md)__ The layer package is used to define propagation and backpropagation of different type of layers. -* __[The Nerv NN Package](doc/nerv_nn.md)__ +* __[The Nerv NN Package](nerv/doc/nerv_nn.md)__ The nn package is for organizing a neural network, it contains __nerv.LayerRepo__, __nerv.ParamRepo__, and __nerv.DAGLayer__. [luaT]:https://github.com/torch/torch7/tree/master/lib/luaT [Torch]:https://github.com/torch diff --git a/nerv/Makefile b/nerv/Makefile index b449f82..55c174c 100644 --- a/nerv/Makefile +++ b/nerv/Makefile @@ -33,7 +33,7 @@ LUA_LIBS := matrix/init.lua io/init.lua init.lua \ layer/init.lua layer/affine.lua layer/sigmoid.lua layer/softmax_ce.lua layer/softmax.lua \ layer/window.lua layer/bias.lua layer/combiner.lua layer/mse.lua layer/affine_recurrent.lua \ nn/init.lua nn/layer_repo.lua nn/param_repo.lua nn/layer_dag.lua \ - io/sgd_buffer.lua + io/sgd_buffer.lua INCLUDE := -I $(LUA_INCDIR) -DLUA_USE_APICHECK #CUDA_BASE := /usr/local/cuda-7.0 @@ -55,7 +55,7 @@ $(OBJ_DIR) $(LUA_DIR) $(OBJ_SUBDIR) $(LUA_SUBDIR) $(INC_SUBDIR): $(OBJ_DIR)/%.o: %.c $(patsubst /%.o,/%.c,$@) gcc -c -o $@ $< $(INCLUDE) -fPIC $(CFLAGS) $(OBJ_DIR)/lib/matrix/cukernel.o: lib/matrix/cukernel.cu - $(NVCC) -c -o $@ $< $(INCLUDE) $(NVCC_FLAGS) + $(NVCC) -c -o $@ $< $(INCLUDE) $(NVCC_FLAGS) $(CFLAGS) $(LUA_DIR)/%.lua: %.lua cp $< $@ diff --git a/nerv/doc/nerv_matrix.md b/nerv/doc/nerv_matrix.md index 22971d2..dfd843d 100644 --- a/nerv/doc/nerv_matrix.md +++ b/nerv/doc/nerv_matrix.md @@ -63,6 +63,12 @@ Copy the content of a __CuMatrix__ `a` to __Matrix__ `self`, they should be of t Copy the content of the __Matrix__ `self` to a __MMatrix__ `a`. * __void Matrix.copy_tod(Matrix self, CuMatrix a)__ Copy the content of the __Matrix__ `self` to a __CuMatrix__ `a`. +* __void Matrix.copy_rows_fromh_by_idx(Matrix self, MMatrix ma, MMatrixInt idx, int idx_begin)__ +`idx` should be a row vector. This function copy the rows of `ma` to `self` according to `idx`, in other words, it assigns `ma[idx[i+idx_begin]]` to `self[i]`. +* __void Matrix.copy_rows_fromd_by_idx(Matrix self, Matrix b, Matrix idx, int idx_begin)__ +`idx` needs to a row vector matrix, it stacks the rows of index `idx` of the __CuMatrix__ `b` and copies to `self`. `idx_begin` is used as an offset in the `idx` index array. +* __void Matrix.update_select_rows(Matrix self, Matrix err, Matrix idx, double alpha, double beta)__ +Update selected rows of `self`, i.e. `self[idx[i]] = self[idx[i]] * (1 - beta * alpha) + alpha * err[i]`. * __void Matrix.add(Matrix self, Matrix ma, Matrix mb, Element_type alpha, Element_type beta)__ It sets the content of __Matrix__ `self` to be `alpha * ma + beta * mb`.__Matrix__ `ma,mb,self` should be of the same size. * __void Matrix.mul(Matrix self, Matrix ma, Matrix mb, Element_type alpha, Element_type beta, [string ta, string tb])__ @@ -81,8 +87,6 @@ Calculate a row-by-row softmax of __Matrix__ `a` and save the result in `self`. Calculate element-wise multiplication of __Matrix__ `ma` and `mb`, store the result in `self`. * __void Matrix.log_elem(Matrix self, Matrix ma)__ Calculate element-wise log of __Matrix__ `ma`, store the result in `self`. -* __void Matrix.copy_rows_fromh_by_idx(Matrix self, MMatrix ma, MMatrixInt idx)__ -`idx` should be a row vector. This function copy the rows of `ma` to `self` according to `idx`, in other words, it assigns `ma[idx[i]]` to `self[i]`. * __void Matrix.expand_frm(Matrix self, Matrix a, int context)__ Treating each row of `a` as speech feature, and do a feature expansion. The `self` should of size `(a.nrow, a.ncol * (context * 2 + 1))`. `self[i]` will be `(a[i-context] a[i-context+1] ... a[i] a[i+1] a[i+context])`. `a[0]` and `a[nrow]` will be copied to extend the index range. * __void Matrix.rearrange_frm(Matrix self, Matrix a, int step)__ @@ -162,4 +166,4 @@ a:mul_elem(fs, fs) print(a) a:log_elem(fs) print(a) -```
\ No newline at end of file +``` diff --git a/nerv/doc/source/conf.py b/nerv/doc/source/conf.py index 8426613..4d6119f 100644 --- a/nerv/doc/source/conf.py +++ b/nerv/doc/source/conf.py @@ -15,6 +15,7 @@ import sys import os import shlex +import alabaster # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -119,7 +120,7 @@ html_theme = 'alabaster' #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +html_theme_path = [alabaster.get_path()] # The name for this set of Sphinx documents. If None, it defaults to # "<project> v<release> documentation". diff --git a/nerv/doc/source/index.rst b/nerv/doc/source/index.rst index e55892b..21cf309 100644 --- a/nerv/doc/source/index.rst +++ b/nerv/doc/source/index.rst @@ -21,7 +21,7 @@ TODO List | Status/ | Task | Assignee | | Priority | | | +==========+==========================================================================+=============+ -| High | Generalize nerv.Matrix to nerv.Tensor (use the same API as Torch Tensor) | Ted Yin | +| High | Generalize nerv.Matrix to nerv.Tensor (use the same API as Torch Tensor) | Mengxiao Bi | +----------+--------------------------------------------------------------------------+-------------+ | High | Development manual: coding style & contribution rules | Ted Yin | +----------+--------------------------------------------------------------------------+-------------+ @@ -31,7 +31,7 @@ TODO List +----------+--------------------------------------------------------------------------+-------------+ | High | support for RNN/LSTM | Tianxing He | +----------+--------------------------------------------------------------------------+-------------+ -| Mid | support for CNN | ? (Ted Yin) | +| High | support for CNN | Mengxiao Bi | +----------+--------------------------------------------------------------------------+-------------+ | Mid | User manual | ALL | +----------+--------------------------------------------------------------------------+-------------+ diff --git a/nerv/examples/lmptb/PTBdata/ptb.test.txt b/nerv/examples/lmptb/PTBdata/ptb.test.txt new file mode 100644 index 0000000..8079cdf --- /dev/null +++ b/nerv/examples/lmptb/PTBdata/ptb.test.txt @@ -0,0 +1,3761 @@ + no it was n't black monday + but while the new york stock exchange did n't fall apart friday as the dow jones industrial average plunged N points most of it in the final hour it barely managed to stay this side of chaos + some circuit breakers installed after the october N crash failed their first test traders say unable to cool the selling panic in both stocks and futures + the N stock specialist firms on the big board floor the buyers and sellers of last resort who were criticized after the N crash once again could n't handle the selling pressure + big investment banks refused to step up to the plate to support the beleaguered floor traders by buying big blocks of stock traders say + heavy selling of standard & poor 's 500-stock index futures in chicago <unk> beat stocks downward + seven big board stocks ual amr bankamerica walt disney capital cities\/abc philip morris and pacific telesis group stopped trading and never resumed + the <unk> has already begun + the equity market was <unk> + once again the specialists were not able to handle the imbalances on the floor of the new york stock exchange said christopher <unk> senior vice president at <unk> securities corp + <unk> james <unk> chairman of specialists henderson brothers inc. it is easy to say the specialist is n't doing his job + when the dollar is in a <unk> even central banks ca n't stop it + speculators are calling for a degree of liquidity that is not there in the market + many money managers and some traders had already left their offices early friday afternoon on a warm autumn day because the stock market was so quiet + then in a <unk> plunge the dow jones industrials in barely an hour surrendered about a third of their gains this year <unk> up a 190.58-point or N N loss on the day in <unk> trading volume + <unk> trading accelerated to N million shares a record for the big board + at the end of the day N million shares were traded + the dow jones industrials closed at N + the dow 's decline was second in point terms only to the <unk> black monday crash that occurred oct. N N + in percentage terms however the dow 's dive was the <unk> ever and the sharpest since the market fell N or N N a week after black monday + the dow fell N N on black monday + shares of ual the parent of united airlines were extremely active all day friday reacting to news and rumors about the proposed $ N billion buy-out of the airline by an <unk> group + wall street 's takeover-stock speculators or risk arbitragers had placed unusually large bets that a takeover would succeed and ual stock would rise + at N p.m. edt came the <unk> news the big board was <unk> trading in ual pending news + on the exchange floor as soon as ual stopped trading we <unk> for a panic said one top floor trader + several traders could be seen shaking their heads when the news <unk> + for weeks the market had been nervous about takeovers after campeau corp. 's cash crunch spurred concern about the prospects for future highly leveraged takeovers + and N minutes after the ual trading halt came news that the ual group could n't get financing for its bid + at this point the dow was down about N points + the market <unk> + arbitragers could n't dump their ual stock but they rid themselves of nearly every rumor stock they had + for example their selling caused trading halts to be declared in usair group which closed down N N to N N delta air lines which fell N N to N N and <unk> industries which sank N to N N + these stocks eventually reopened + but as panic spread speculators began to sell blue-chip stocks such as philip morris and international business machines to offset their losses + when trading was halted in philip morris the stock was trading at N down N N while ibm closed N N lower at N + selling <unk> because of waves of automatic stop-loss orders which are triggered by computer when prices fall to certain levels + most of the stock selling pressure came from wall street professionals including computer-guided program traders + traders said most of their major institutional investors on the other hand sat tight + now at N one of the market 's post-crash reforms took hold as the s&p N futures contract had plunged N points equivalent to around a <unk> drop in the dow industrials + under an agreement signed by the big board and the chicago mercantile exchange trading was temporarily halted in chicago + after the trading halt in the s&p N pit in chicago waves of selling continued to hit stocks themselves on the big board and specialists continued to <unk> prices down + as a result the link between the futures and stock markets <unk> apart + without the <unk> of stock-index futures the barometer of where traders think the overall stock market is headed many traders were afraid to trust stock prices quoted on the big board + the futures halt was even <unk> by big board floor traders + it <unk> things up said one major specialist + this confusion effectively halted one form of program trading stock index arbitrage that closely links the futures and stock markets and has been blamed by some for the market 's big swings + in a stock-index arbitrage sell program traders buy or sell big baskets of stocks and offset the trade in futures to lock in a price difference + when the airline information came through it <unk> every model we had for the marketplace said a managing director at one of the largest program-trading firms + we did n't even get a chance to do the programs we wanted to do + but stocks kept falling + the dow industrials were down N points at N p.m. before the <unk> halt + at N p.m. at the end of the cooling off period the average was down N points + meanwhile during the the s&p trading halt s&p futures sell orders began <unk> up while stocks in new york kept falling sharply + big board chairman john j. phelan said yesterday the circuit breaker worked well <unk> + i just think it 's <unk> at this point to get into a debate if index arbitrage would have helped or hurt things + under another post-crash system big board president richard <unk> mr. phelan was flying to <unk> as the market was falling was talking on an <unk> hot line to the other exchanges the securities and exchange commission and the federal reserve board + he <unk> out at a high-tech <unk> center on the floor of the big board where he could watch <unk> on prices and pending stock orders + at about N p.m. edt s&p futures resumed trading and for a brief time the futures and stock markets started to come back in line + buyers stepped in to the futures pit + but the <unk> of s&p futures sell orders weighed on the market and the link with stocks began to fray again + at about N the s&p market <unk> to still another limit of N points down and trading was locked again + futures traders say the s&p was <unk> that the dow could fall as much as N points + during this time small investors began ringing their brokers wondering whether another crash had begun + at prudential-bache securities inc. which is trying to cater to small investors some <unk> brokers thought this would be the final <unk> + that 's when george l. ball chairman of the prudential insurance co. of america unit took to the internal <unk> system to declare that the plunge was only mechanical + i have a <unk> that this particular decline today is something more <unk> about less + it would be my <unk> to advise clients not to sell to look for an opportunity to buy mr. ball told the brokers + at merrill lynch & co. the nation 's biggest brokerage firm a news release was prepared <unk> merrill lynch comments on market drop + the release cautioned that there are significant differences between the current environment and that of october N and that there are still attractive investment opportunities in the stock market + however jeffrey b. lane president of shearson lehman hutton inc. said that friday 's plunge is going to set back relations with customers because it <unk> the concern of volatility + and i think a lot of people will <unk> on program trading + it 's going to bring the debate right back to the <unk> + as the dow average ground to its final N loss friday the s&p pit stayed locked at its <unk> trading limit + jeffrey <unk> of program trader <unk> investment group said N s&p contracts were for sale on the close the equivalent of $ N million in stock + but there were no buyers + while friday 's debacle involved mainly professional traders rather than investors it left the market vulnerable to continued selling this morning traders said + stock-index futures contracts settled at much lower prices than indexes of the stock market itself + at those levels stocks are set up to be <unk> by index arbitragers who lock in profits by buying futures when futures prices fall and simultaneously sell off stocks + but nobody knows at what level the futures and stocks will open today + the <unk> between the stock and futures markets friday will undoubtedly cause renewed debate about whether wall street is properly prepared for another crash situation + the big board 's mr. <unk> said our <unk> performance was good + but the exchange will look at the performance of all specialists in all stocks + obviously we 'll take a close look at any situation in which we think the <unk> obligations were n't met he said + see related story fed ready to <unk> big funds wsj oct. N N + but specialists complain privately that just as in the N crash the <unk> firms big investment banks that support the market by trading big blocks of stock stayed on the sidelines during friday 's <unk> + mr. phelan said it will take another day or two to analyze who was buying and selling friday + concerning your sept. N page-one article on prince charles and the <unk> it 's a few hundred years since england has been a kingdom + it 's now the united kingdom of great britain and northern ireland <unk> <unk> northern ireland scotland and oh yes england too + just thought you 'd like to know + george <unk> + ports of call inc. reached agreements to sell its remaining seven aircraft to buyers that were n't disclosed + the agreements bring to a total of nine the number of planes the travel company has sold this year as part of a restructuring + the company said a portion of the $ N million realized from the sales will be used to repay its bank debt and other obligations resulting from the currently suspended <unk> operations + earlier the company announced it would sell its aging fleet of boeing co. <unk> because of increasing maintenance costs + a consortium of private investors operating as <unk> funding co. said it has made a $ N million cash bid for most of l.j. hooker corp. 's real-estate and <unk> holdings + the $ N million bid includes the assumption of an estimated $ N million in secured liabilities on those properties according to those making the bid + the group is led by jay <unk> chief executive officer of <unk> investment corp. in <unk> and a. boyd simpson chief executive of the atlanta-based simpson organization inc + mr. <unk> 's company specializes in commercial real-estate investment and claims to have $ N billion in assets mr. simpson is a developer and a former senior executive of l.j. hooker + the assets are good but they require more money and management than can be provided in l.j. hooker 's current situation said mr. simpson in an interview + hooker 's philosophy was to build and sell + we want to build and hold + l.j. hooker based in atlanta is operating with protection from its creditors under chapter N of the u.s. bankruptcy code + its parent company hooker corp. of sydney australia is currently being managed by a court-appointed provisional <unk> + sanford <unk> chief executive of l.j. hooker said yesterday in a statement that he has not yet seen the bid but that he would review it and bring it to the attention of the creditors committee + the $ N million bid is estimated by mr. simpson as representing N N of the value of all hooker real-estate holdings in the u.s. + not included in the bid are <unk> teller or b. altman & co. l.j. hooker 's department-store chains + the offer covers the massive N <unk> forest fair mall in cincinnati the N <unk> <unk> fashion mall in columbia s.c. and the N <unk> <unk> town center mall in <unk> <unk> + the <unk> mall opened sept. N with a <unk> 's <unk> as its <unk> the columbia mall is expected to open nov. N + other hooker properties included are a <unk> office tower in <unk> atlanta expected to be completed next february vacant land sites in florida and ohio l.j. hooker international the commercial real-estate brokerage company that once did business as merrill lynch commercial real estate plus other shopping centers + the consortium was put together by <unk> <unk> the london-based investment banking company that is a subsidiary of security pacific corp + we do n't anticipate any problems in raising the funding for the bid said <unk> campbell the head of mergers and acquisitions at <unk> <unk> in an interview + <unk> <unk> is acting as the consortium 's investment bankers + according to people familiar with the consortium the bid was <unk> project <unk> a reference to the film <unk> in which a <unk> played by actress <unk> <unk> is saved from a <unk> businessman by a police officer named john <unk> + l.j. hooker was a small <unk> company based in atlanta in N when mr. simpson was hired to push it into commercial development + the company grew modestly until N when a majority position in hooker corp. was acquired by australian developer george <unk> currently hooker 's chairman + mr. <unk> <unk> to launch an ambitious but <unk> $ N billion acquisition binge that included <unk> teller and b. altman & co. as well as majority positions in merksamer jewelers a sacramento chain <unk> inc. the <unk> retailer and <unk> inc. the southeast department-store chain + eventually mr. simpson and mr. <unk> had a falling out over the direction of the company and mr. simpson said he resigned in N + since then hooker corp. has sold its interest in the <unk> chain back to <unk> 's management and is currently attempting to sell the b. altman & co. chain + in addition robert <unk> chief executive of the <unk> chain is seeking funds to buy out the hooker interest in his company + the merksamer chain is currently being offered for sale by first boston corp + reached in <unk> mr. <unk> said that he believes the various hooker <unk> can become profitable with new management + these are n't mature assets but they have the potential to be so said mr. <unk> + managed properly and with a long-term outlook these can become investment-grade quality properties + canadian <unk> production totaled N metric tons in the week ended oct. N up N N from the preceding week 's total of N tons statistics canada a federal agency said + the week 's total was up N N from N tons a year earlier + the <unk> total was N tons up N N from N tons a year earlier + the treasury plans to raise $ N million in new cash thursday by selling about $ N billion of 52-week bills and <unk> $ N billion of maturing bills + the bills will be dated oct. N and will mature oct. N N + they will be available in minimum denominations of $ N + bids must be received by N p.m. edt thursday at the treasury or at federal reserve banks or branches + as small investors <unk> their mutual funds with phone calls over the weekend big fund managers said they have a strong defense against any wave of withdrawals cash + unlike the weekend before black monday the funds were n't <unk> with heavy withdrawal requests + and many fund managers have built up cash levels and say they will be buying stock this week + at fidelity investments the nation 's largest fund company telephone volume was up sharply but it was still at just half the level of the weekend preceding black monday in N + the boston firm said <unk> redemptions were running at less than one-third the level two years ago + as of yesterday afternoon the redemptions represented less than N N of the total cash position of about $ N billion of fidelity 's stock funds + two years ago there were massive redemption levels over the weekend and a lot of fear around said c. bruce <unk> who runs fidelity investments ' $ N billion <unk> fund + this feels more like a <unk> deal + people are n't <unk> + the test may come today + friday 's stock market sell-off came too late for many investors to act + some shareholders have held off until today because any fund exchanges made after friday 's close would take place at today 's closing prices + stock fund redemptions during the N debacle did n't begin to <unk> until after the market opened on black monday + but fund managers say they 're ready + many have raised cash levels which act as a buffer against steep market declines + mario <unk> for instance holds cash positions well above N N in several of his funds + windsor fund 's john <unk> and mutual series ' michael price said they had raised their cash levels to more than N N and N N respectively this year + even peter lynch manager of fidelity 's $ N billion <unk> fund the nation 's largest stock fund built up cash to N N or $ N million + one rea |