[irstlm] 13/78: code cleanup
Giulio Paci
giuliopaci-guest at moszumanska.debian.org
Tue May 17 07:47:01 UTC 2016
This is an automated email from the git hooks/post-receive script.
giuliopaci-guest pushed a commit to tag adaptiveLM.v0.10
in repository irstlm.
commit 0897e08e5f1cfa234ca841475c5570cfc4168511
Author: Nicola Bertoldi <bertoldi at fbk.eu>
Date: Sun Nov 8 16:29:00 2015 +0100
code cleanup
---
src/lmContainer.h | 5 +-
src/lmInterpolation.cpp | 40 +--------
src/lmInterpolation.h | 8 +-
src/lmclass.h | 22 ++---
src/lmmacro.cpp | 18 +---
src/lmmacro.h | 9 +-
src/lmtable.cpp | 221 ++----------------------------------------------
src/lmtable.h | 23 ++---
8 files changed, 37 insertions(+), 309 deletions(-)
diff --git a/src/lmContainer.h b/src/lmContainer.h
index 4f18c3c..8881617 100644
--- a/src/lmContainer.h
+++ b/src/lmContainer.h
@@ -129,9 +129,8 @@ namespace irstlm {
virtual double clprob(ngram ng, double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr, unsigned int* statesize, bool* extendible) { return clprob(ng, bow, bol, maxsuffidx, maxsuffptr, statesize, extendible, NULL); };
virtual double clprob(ngram ng, double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr, unsigned int* statesize, bool* extendible, double* lastbow) { return clprob(ng, bow, bol, maxsuffidx, maxsuffptr, statesize, extendible, lastbow); }
- virtual double clprob(int* ng, int ngsize=NULL, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL,bool* extendible=NULL, double* lastbow=NULL) {
- VERBOSE(3,"virtual double lmContainer::clprob(int* ng, int ngsize, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx, char** maxsuffptr=NULL, unsigned int* statesize=NULL,bool* extendible=NULL, double* lastbow=NULL)\n");
-
+ virtual double clprob(int* ng, int ngsize=NULL, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL,bool* extendible=NULL, double* lastbow=NULL)
+ {
//create the actual ngram
ngram ong(getDict());
ong.pushc(ng,ngsize);
diff --git a/src/lmInterpolation.cpp b/src/lmInterpolation.cpp
index dbc9c5f..a8badab 100644
--- a/src/lmInterpolation.cpp
+++ b/src/lmInterpolation.cpp
@@ -140,7 +140,6 @@ namespace irstlm {
}
//return log10 prob of an ngram
- // double lmInterpolation::clprob(ngram ng, double* bow,int* bol,char** maxsuffptr,unsigned int* statesize,bool* extendible)
double lmInterpolation::clprob(ngram ng, double* bow,int* bol,ngram_state_t* maxsuffidx, char** maxsuffptr,unsigned int* statesize,bool* extendible, double* lastbow)
{
@@ -223,22 +222,8 @@ namespace irstlm {
return log10(pr);
}
- /*
- // double lmInterpolation::clprob(int* codes, int sz, double* bow,int* bol,char** maxsuffptr,unsigned int* statesize,bool* extendible)
- double lmInterpolation::clprob(int* codes, int sz, double* bow,int* bol,ngram_state_t* maxsuffidx,char** maxsuffptr,unsigned int* statesize,bool* extendible)
- {
-
- //create the actual ngram
- ngram ong(dict);
- ong.pushc(codes,sz);
- MY_ASSERT (ong.size == sz);
-
- // return clprob(ong, bow, bol, maxsuffptr, statesize, extendible);
- return clprob(ong, bow, bol, maxsuffidx, maxsuffptr, statesize, extendible);
- }
- */
-
- const char *lmInterpolation::cmaxsuffptr(ngram ng, unsigned int* statesize){
+ const char *lmInterpolation::cmaxsuffptr(ngram ng, unsigned int* statesize)
+ {
char *maxsuffptr=NULL;
unsigned int _statesize=0,actualstatesize=0;
@@ -278,16 +263,6 @@ namespace irstlm {
return maxsuffptr;
}
- /*
- const char *lmInterpolation::cmaxsuffptr(int* codes, int sz, unsigned int* statesize)
- {
- //create the actual ngram
- ngram ong(dict);
- ong.pushc(codes,sz);
- MY_ASSERT (ong.size == sz);
- return cmaxsuffptr(ong, statesize);
- }
- */
ngram_state_t lmInterpolation::cmaxsuffidx(ngram ng, unsigned int* statesize)
{
ngram_state_t maxsuffidx=0;
@@ -328,17 +303,6 @@ namespace irstlm {
return maxsuffidx;
}
-
- /*
- ngram_state_t lmInterpolation::cmaxsuffidx(int* codes, int sz, unsigned int* statesize)
- {
- //create the actual ngram
- ngram ong(dict);
- ong.pushc(codes,sz);
- MY_ASSERT (ong.size == sz);
- return cmaxsuffidx(ong, statesize);
- }
- */
double lmInterpolation::setlogOOVpenalty(int dub)
{
diff --git a/src/lmInterpolation.h b/src/lmInterpolation.h
index 12a7add..2f12265 100644
--- a/src/lmInterpolation.h
+++ b/src/lmInterpolation.h
@@ -73,13 +73,9 @@ namespace irstlm {
lmContainer* load_lm(int i, int memmap, float nlf, float dlf);
virtual double clprob(ngram ng, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL);
- // double clprob(int* ng, int ngsize, double* bow=NULL,int* bol=NULL,ngram_state_t* maxsuffidx=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL);
-
- const char *cmaxsuffptr(ngram ong, unsigned int* size=NULL);
- // const char *cmaxsuffptr(int* codes, int sz, unsigned int* size=NULL);
- ngram_state_t cmaxsuffidx(ngram ong, unsigned int* size=NULL);
- // ngram_state_t cmaxsuffidx(int* codes, int sz, unsigned int* size=NULL);
+ virtual const char *cmaxsuffptr(ngram ong, unsigned int* size=NULL);
+ virtual ngram_state_t cmaxsuffidx(ngram ong, unsigned int* size=NULL);
int maxlevel() const {
return maxlev;
diff --git a/src/lmclass.h b/src/lmclass.h
index a22164d..3fcfba0 100644
--- a/src/lmclass.h
+++ b/src/lmclass.h
@@ -75,23 +75,15 @@ namespace irstlm {
virtual void load(const std::string &filename,int mmap=0);
+ /*
+ virtual double lprob(ngram ng, double* bow=NULL, int* bol=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL) { return lprob(ng,bow,bol,NULL,maxsuffptr,statesize,extendible,lastbow); };
+ virtual double clprob(ngram ng,double* bow=NULL,int* bol=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL) { return clprob(ng,bow,bol,NULL,maxsuffptr,statesize,extendible); };
+ virtual double clprob(int* ng, int ngsize, double* bow=NULL,int* bol=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL) { return clprob(ng,ngsize,bow,bol,NULL,maxsuffptr,statesize,extendible); };
+ */
- // virtual double lprob(ngram ng, double* bow=NULL, int* bol=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL) { return lprob(ng,bow,bol,NULL,maxsuffptr,statesize,extendible,lastbow); };
- // virtual double clprob(ngram ng,double* bow=NULL,int* bol=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL) { return clprob(ng,bow,bol,NULL,maxsuffptr,statesize,extendible); };
+ virtual double lprob(ngram ng, double* bow=NULL,int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL);
+ virtual double clprob(ngram ng, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL) { return lprob(ng, bow, bol, maxsuffidx, maxsuffptr, statesize, extendible, lastbow); };
- // double clprob(int* ng, int ngsize, double* bow=NULL,int* bol=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL) { return clprob(ng,ngsize,bow,bol,NULL,maxsuffptr,statesize,extendible); };
-
- double lprob(ngram ng, double* bow=NULL,int* bol=NULL,ngram_state_t* maxsuffidx=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL, double* lastbow=NULL);
- double clprob(ngram ng,double* bow=NULL,int* bol=NULL,ngram_state_t* maxsuffidx=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL, double* lastbow=NULL) {
- return lprob(ng,bow,bol,maxsuffidx,maxsuffptr,statesize,extendible,lastbow);
- };
- /*
- double clprob(int* ng, int ngsize, double* bow=NULL,int* bol=NULL,ngram_state_t* maxsuffidx=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL) {
- ngram ong(getDict());
- ong.pushc(ng,ngsize);
- return lprob(ong,bow,bol,maxsuffidx,maxsuffptr,statesize,extendible);
- };
- */
inline bool is_OOV(int code) {
//a word is consisdered OOV if its mapped value is OOV
return lmtable::is_OOV(getMap(code));
diff --git a/src/lmmacro.cpp b/src/lmmacro.cpp
index 0f64477..1ca4aad 100644
--- a/src/lmmacro.cpp
+++ b/src/lmmacro.cpp
@@ -306,7 +306,8 @@ namespace irstlm {
}
- double lmmacro::lprob(ngram micro_ng)
+// double lmmacro::lprob(ngram micro_ng)
+ double lmmacro::lprob(ngram micro_ng, double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr, unsigned int* statesize, bool* extendible, double* lastbow)
{
VERBOSE(2,"lmmacro::lprob, parameter = <" << micro_ng << ">\n");
@@ -322,23 +323,12 @@ namespace irstlm {
// ask LM with macro
double prob;
- prob = lmtable::lprob(macro_ng);
+ prob = lmtable::lprob(macro_ng, bow, bol, maxsuffidx, maxsuffptr, statesize, extendible, lastbow);
VERBOSE(3,"prob = " << prob << "\n");
return prob;
};
- /*
- //double lmmacro::clprob(int* codes, int sz, double* bow, int* bol, char** state,unsigned int* statesize,bool* extendible)
- double lmmacro::clprob(int* codes, int sz, double* bow, int* bol, ngram_state_t* ngramstate, char** state,unsigned int* statesize,bool* extendible, double* lastbow)
- {
- ngram micro_ng(getDict());
- micro_ng.pushc(codes,sz);
- // return clprob(micro_ng,bow,bol,state,statesize,extendible);
- return clprob(micro_ng,bow,bol,ngramstate,state,statesize,extendible, lastbow);
- }
- */
-
- // double lmmacro::clprob(ngram micro_ng, double* bow, int* bol, char** state,unsigned int* statesize,bool* extendible)
+
double lmmacro::clprob(ngram micro_ng, double* bow, int* bol, ngram_state_t* ngramstate, char** state,unsigned int* statesize,bool* extendible, double* lastbow)
{
diff --git a/src/lmmacro.h b/src/lmmacro.h
index 9cdec56..5ff8421 100644
--- a/src/lmmacro.h
+++ b/src/lmmacro.h
@@ -76,15 +76,12 @@ namespace irstlm {
virtual void load(const std::string &filename,int mmap=0);
- virtual double lprob(ngram ng);
- virtual double clprob(ngram ng,double* bow=NULL,int* bol=NULL,ngram_state_t* maxsuffidx=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL, double* lastbow=NULL);
- // double clprob(int* ng, int ngsize, double* bow=NULL,int* bol=NULL,ngram_state_t* maxsuffidx=NULL,char** maxsuffptr=NULL,unsigned int* statesize=NULL,bool* extendible=NULL);
+ virtual double lprob(ngram ng, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL);
+ virtual double clprob(ngram ng, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL);
virtual const char *maxsuffptr(ngram ong, unsigned int* size=NULL);
- // const char *cmaxsuffptr(ngram ong, unsigned int* size=NULL);
virtual ngram_state_t maxsuffidx(ngram ong, unsigned int* size=NULL);
- // ngram_state_t cmaxsuffidx(ngram ong, unsigned int* size=NULL);
-
+
void map(ngram *in, ngram *out);
void One2OneMapping(ngram *in, ngram *out);
void Micro2MacroMapping(ngram *in, ngram *out);
diff --git a/src/lmtable.cpp b/src/lmtable.cpp
index 3865230..948bfd1 100644
--- a/src/lmtable.cpp
+++ b/src/lmtable.cpp
@@ -2249,7 +2249,7 @@ namespace irstlm {
//maxsuffptr returns the largest suffix of an n-gram that is contained
//in the LM table. This can be used as a compact representation of the
- //(n-1)-gram state of a n-gram LM. if the input k-gram has k>=n then it
+ //(n-1)-gram state of a n-gram LM. If the input k-gram has k>=n then it
//is trimmed to its n-1 suffix.
//non recursive version
@@ -2347,62 +2347,12 @@ namespace irstlm {
return (char *)maxsuffptr(ong,size);
#endif
}
+
- /*
- //this function simulates the cmaxsuffptr(ngram, ...) but it takes as input an array of codes instead of the ngram
- const char *lmtable::cmaxsuffptr(int* codes, int sz, unsigned int* size)
- {
- VERBOSE(3,"const char *lmtable::cmaxsuffptr(int* codes, int sz, unsigned int* size)\n");
-
- if (sz==0) {
- if (size!=NULL) *size=0;
- return (char*) NULL;
- }
-
- if (sz>maxlev) sz=maxlev; //adjust n-gram level to table size
-
-#ifdef PS_CACHE_ENABLE
- //cache hit
- prob_and_state_t pst;
-
- //cache hit
- // if (prob_and_state_cache && sz==maxlev && prob_and_state_cache->get(codes,pst)) {
- if (prob_and_state_cache[sz] && prob_and_state_cache[sz]->get(codes,pst)) {
- if (size) *size = pst.statesize;
- return pst.state;
- }
-
- //create the actual ngram
- ngram ong(dict);
- ong.pushc(codes,sz);
- MY_ASSERT (ong.size == sz);
-
- //cache miss
- unsigned int isize; //internal state size variable
- char* found=(char *)maxsuffptr(ong,&isize);
-
- //cache insert
- //IMPORTANT: this function updates only two fields (state, statesize) of the entry of the cache; the reminaing fields (logpr, bow, bol, extendible) are undefined; hence, it should not be used before the corresponding clprob()
- if (ong.size>=maxlev) ong.size=maxlev;
- // if (prob_and_state_cache && ong.size==maxlev) {
- if (prob_and_state_cache[sz]) {
- pst.state=found;
- pst.statesize=isize;
- // prob_and_state_cache->add(ong.wordp(maxlev),pst);
- prob_and_state_cache[sz]->add(ong.wordp(ong.size),pst);
- }
- if (size!=NULL) *size=isize;
- return found;
-#else
- //create the actual ngram
- ngram ong(dict);
- ong.pushc(codes,sz);
- MY_ASSERT (ong.size == sz);
- return maxsuffptr(ong,size);
-#endif
- }
- */
-
+ //maxsuffidx returns an index of the largest of an n-gram that is contained
+ //in the LM table. This can be used as a compact representation of the
+ //(n-1)-gram state of a n-gram LM. If the input k-gram has k>=n then it
+ //is trimmed to its n-1 suffix.
//non recursive version
ngram_state_t lmtable::maxsuffidx(ngram ong, unsigned int* size)
{
@@ -2439,7 +2389,6 @@ namespace irstlm {
if (size!=NULL) *size=ong.size; //will return the largest found ong.size
-
for (ngram ng=ong; ng.size>0; ng.size--) {
if (get(ng,ng.size,ng.size)) {
// if (ng.succ==0) (*size)--;
@@ -2514,60 +2463,6 @@ namespace irstlm {
#endif
}
- /*
- //this function simulates the cmaxsuffptr(ngram, ...) but it takes as input an array of codes instead of the ngram
- ngram_state_t lmtable::cmaxsuffidx(int* codes, int sz, unsigned int* size)
- {
- VERBOSE(3,"const char *lmtable::cmaxsuffptr(int* codes, int sz, unsigned int* size)\n");
-
- if (sz==0) {
- if (size!=NULL) *size=0;
- return 0;
- }
-
- if (sz>maxlev) sz=maxlev; //adjust n-gram level to table size
-
-#ifdef PS_CACHE_ENABLE
- //cache hit
- prob_and_state_t pst;
-
- //cache hit
- // if (prob_and_state_cache && sz==maxlev && prob_and_state_cache->get(codes,pst)) {
- if (prob_and_state_cache[sz] && prob_and_state_cache[sz]->get(codes,pst)) {
- if (size) *size = pst.statesize;
- return pst.ngramstate;
- }
-
- //create the actual ngram
- ngram ong(dict);
- ong.pushc(codes,sz);
- MY_ASSERT (ong.size == sz);
-
- //cache miss
- unsigned int isize; //internal state size variable
- ngram_state_t msidx = maxsuffidx(ong,&isize);
-
- //cache insert
- //IMPORTANT: this function updates only two fields (ngramstate, statesize) of the entry of the cache; the reminaing fields (logpr, bow, bol, extendible) are undefined; hence, it should not be used before the corresponding clprob()
- if (ong.size>=maxlev) ong.size=maxlev;
- // if (prob_and_state_cache && ong.size==maxlev) {
- if (prob_and_state_cache[sz]) {
- pst.ngramstate=msidx;
- pst.statesize=isize;
- prob_and_state_cache[sz]->add(ong.wordp(ong.size),pst);
- }
- if (size!=NULL) *size=isize;
- return msidx;
-#else
- //create the actual ngram
- ngram ong(dict);
- ong.pushc(codes,sz);
- MY_ASSERT (ong.size == sz);
- return maxsuffidx(ong,size);
-#endif
- }
- */
-
//returns log10prob of n-gram
//bow: backoff weight
//bol: backoff level
@@ -2582,7 +2477,6 @@ namespace irstlm {
double lmtable::lprob(ngram ong,double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr,unsigned int* statesize,bool* extendible, double *lastbow)
{
VERBOSE(3," lmtable::lprob(ngram) ong |" << ong << "|\n" << std::endl);
- VERBOSE(3," lmtable::lprob(ngram) ong.size |" << ong.size << "|\n" << std::endl);
if (ong.size==0){ //sanity check
if (maxsuffptr) *maxsuffptr=NULL;
@@ -2614,7 +2508,6 @@ namespace irstlm {
if (maxsuffidx){
int ndsz=nodesize(tbltype[isize]);
*maxsuffidx = (ngram_state_t) ( ((table_pos_t) (ing.path[isize]) - (table_pos_t) table[isize] ) / ndsz ) + tb_offset[isize] + 1; //added 1 to distinguish from zero-ngram
- VERBOSE(3,"lmtable::lprob(ngram) ing:|" << ing << "| isize:|" << isize << "| ing.path[isize]:|" << ing.path[isize] << "| tb_offset[_level]:|" << tb_offset[isize] << "| *maxsuffidx:|" << *maxsuffidx << "|" << std::endl);
}
if (extendible) *extendible=succrange(ing.path[ing.lev],ing.lev)>0;
if (lastbow) *lastbow=(double) (isQtable?Bcenters[ing.lev][(qfloat_t)ing.bow]:ing.bow);
@@ -2653,15 +2546,11 @@ namespace irstlm {
MY_ASSERT((extendible == NULL) || (extendible && *extendible==false));
// MY_ASSERT(lastbow==NULL);
for (ngram ng=ong; ng.size>0; ng.size--) {
- VERBOSE(3,"lmtable::lprob(ngram) ng:|" << ng << "|" << std::endl);
if (get(ng,ng.size,ng.size)) {
- VERBOSE(3,"lmtable::lprob(ngram) ng:|" << ng << "| inside if get" << std::endl);
iprob=ng.prob;
lpr = (double)(isQtable?Pcenters[ng.size][(qfloat_t)iprob]:iprob);
if (*ng.wordp(1)==dict->oovcode()) lpr-=logOOVpenalty; //add OOV penalty
if (maxsuffptr || statesize) { //one extra step is needed if ng.size=ong.size
-
- VERBOSE(3,"lmtable::lprob(ngram) ng:|" << ng << "| inside if maxsuffptr" << std::endl);
if (ong.size==ng.size) {
ng.size--;
get(ng,ng.size,ng.size);
@@ -2669,8 +2558,7 @@ namespace irstlm {
if (statesize) *statesize=ng.size;
if (maxsuffptr) *maxsuffptr=ng.link; //we should check ng.link != NULL
- size_t isize=ng.size;
- VERBOSE(3,"lmtable::lprob(ngram) ng:|" << ng << "| isize:|" << isize << "|" << std::endl);
+ size_t isize=ng.size;
if (maxsuffidx){
int ndsz=nodesize(tbltype[isize]);
@@ -2678,15 +2566,10 @@ namespace irstlm {
if (ng.link){
*maxsuffidx = (ngram_state_t) ( ((table_pos_t) (ng.link) - (table_pos_t) table[isize]) / ndsz ) + tb_offset[isize] + 1; //added 1 to distinguish from zero-ngram
}
-
- VERBOSE(3,"lmtable::lprob(ngram) ng:|" << ng << "| *maxsuffidx:|" << *maxsuffidx << "|" << std::endl);
}
}
- VERBOSE(3,"lmtable::lprob(ngram) returning (rbow+lpr):|" << (rbow+lpr) << "|" << std::endl);
return rbow+lpr;
} else {
- VERBOSE(3,"lmtable::lprob(ngram) ng:|" << ng << "| inside else get" << std::endl);
- VERBOSE(3,"lmtable::lprob(ngram) ng.size:|" << ng.size << "|" << std::endl);
if (ng.size==1) { //means a real unknow word!
if (statesize) *statesize=0;
if (maxsuffptr) *maxsuffptr=NULL; //default stateptr for zero-gram!
@@ -2706,15 +2589,12 @@ namespace irstlm {
}
}
- VERBOSE(3,"lmtable::lprob(ngram) ng:|" << ng << "| END " << std::endl);
-
}
}
MY_ASSERT(0); //never pass here!!!
return 1.0;
}
-
//return log10 probsL use cache memory
double lmtable::clprob(ngram ong,double* bow, int* bol, ngram_state_t* ngramstate, char** state, unsigned int* statesize, bool* extendible, double* lastbow)
{
@@ -2769,7 +2649,7 @@ namespace irstlm {
if (ngramstate) *ngramstate = pst_add.ngramstate;
if (statesize) *statesize = pst_add.statesize;
if (extendible) *extendible = pst_add.extendible;
- if (extendible) *lastbow = pst_add.lastbow;
+ if (lastbow) *lastbow = pst_add.lastbow;
// if (prob_and_state_cache && ong.size==maxlev) {
@@ -2784,91 +2664,6 @@ namespace irstlm {
#endif
};
- /*
- //return log10 probsL use cache memory
- //this function simulates the clprob(ngram, ...) but it takes as input an array of codes instead of the ngram
- double lmtable::clprob(int* codes, int sz, double* bow, int* bol, ngram_state_t* ngramstate, char** state,unsigned int* statesize,bool* extendible, double* lastbow)
- {
- VERBOSE(3," double lmtable::clprob(int* codes, int sz, double* bow, int* bol, ngram_state_t* ngramstate, char** state, unsigned int* statesize, bool* extendible, double* lastbow)\n");
- #ifdef TRACE_CACHELM
- // if (probcache && sz==maxlev && sentence_id>0) {
- if (probcache && sentence_id>0) {
- *cacheout << sentence_id << "\n";
- //print the codes of the vector ng
- }
- #endif
-
- if (sz==0) {
- if (statesize!=NULL) *statesize=0;
- if (state!=NULL) *state=NULL;
- if (ngramstate!=NULL) *ngramstate=NULL;
- if (extendible!=NULL) *extendible=false;
- if (lastbow!=NULL) *lastbow=false;
- return 0.0;
- }
-
- if (sz>maxlev) sz=maxlev; //adjust n-gram level to table size
-
- #ifdef PS_CACHE_ENABLE
- double logpr;
-
- //cache hit
- prob_and_state_t pst_get;
-
- // if (prob_and_state_cache && sz==maxlev && prob_and_state_cache->get(codes,pst_get)) {
- if (prob_and_state_cache[sz] && prob_and_state_cache[sz]->get(codes,pst_get)) {
-
- logpr=pst_get.logpr;
- if (bow) *bow = pst_get.bow;
- if (bol) *bol = pst_get.bol;
- if (state) *state = pst_get.state;
- if (ngramstate) *ngramstate = pst_get.ngramstate;
- if (statesize) *statesize = pst_get.statesize;
- if (extendible) *extendible = pst_get.extendible;
- if (lastbow) *lastbow = pst_get.lastbow;
-
- return logpr;
- }
-
-
- //create the actual ngram
- ngram ong(dict);
- ong.pushc(codes,sz);
- MY_ASSERT (ong.size == sz);
-
- //cache miss
- prob_and_state_t pst_add;
- // logpr = pst_add.logpr = lmtable::lprob(ong, &(pst_add.bow), &(pst_add.bol), &(pst_add.state), &(pst_add.statesize), &(pst_add.extendible), &(pst_add.lastbow));
- logpr = pst_add.logpr = lmtable::lprob(ong, &(pst_add.bow), &(pst_add.bol), &(pst_add.ngramstate), &(pst_add.state), &(pst_add.statesize), &(pst_add.extendible), &(pst_add.lastbow));
-
-
- if (bow) *bow = pst_add.bow;
- if (bol) *bol = pst_add.bol;
- if (state) *state = pst_add.state;
- if (ngramstate) *ngramstate = pst_add.ngramstate;
- if (statesize) *statesize = pst_add.statesize;
- if (extendible) *extendible = pst_add.extendible;
- if (lastbow) *lastbow = pst_add.lastbow;
-
-
- // if (prob_and_state_cache && ong.size==maxlev) {
- // prob_and_state_cache->add(ong.wordp(maxlev),pst_add);
- // }
- if (prob_and_state_cache[sz]) {
- prob_and_state_cache[sz]->add(ong.wordp(ong.size),pst_add);
- }
- return logpr;
- #else
-
- //create the actual ngram
- ngram ong(dict);
- ong.pushc(codes,sz);
- MY_ASSERT (ong.size == sz);
- return lmtable::lprob(ong, bow, bol, ngramstate, state, statesize, extendible,lastbow);
- #endif
- };
- */
-
int lmtable::succrange(node ndp,int level,table_entry_pos_t* isucc,table_entry_pos_t* esucc)
{
table_entry_pos_t first,last;
diff --git a/src/lmtable.h b/src/lmtable.h
index 6bb707f..e7cba2d 100644
--- a/src/lmtable.h
+++ b/src/lmtable.h
@@ -314,7 +314,7 @@ namespace irstlm {
int reload(std::set<string> words);
void filter(const char* /* unused parameter: lmfile */) {};
-
+ /*
virtual double lprob(ngram ng){ return lprob(ng, NULL, NULL, NULL, NULL, NULL, NULL, NULL); }
virtual double lprob(ngram ng, double* bow){ return lprob(ng, bow, NULL, NULL, NULL, NULL, NULL, NULL); }
virtual double lprob(ngram ng, double* bow, int* bol){ return lprob(ng, bow, bol, NULL, NULL, NULL, NULL, NULL); }
@@ -326,14 +326,16 @@ namespace irstlm {
virtual double lprob(ngram ng, double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr){ return lprob(ng, bow, bol, maxsuffidx, maxsuffptr, NULL, NULL, NULL); }
virtual double lprob(ngram ng, double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr, unsigned int* statesize){ return lprob(ng, bow, bol, maxsuffidx, maxsuffptr, statesize, NULL, NULL); }
virtual double lprob(ngram ng, double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr, unsigned int* statesize, bool* extendible){ return lprob(ng, bow, bol, maxsuffidx, maxsuffptr, statesize, extendible, NULL); }
-
-
-// virtual double lprob(ngram ng, double* bow, int* bol, char** maxsuffptr, unsigned int* statesize, bool* extendible, double* lastbow){ return lprob(ng, bow, bol, NULL, maxsuffptr, statesize, extendible, lastbow); }
- virtual double lprob(ngram ng, double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr, unsigned int* statesize, bool* extendible, double* lastbow);
+ */
+
+ virtual double lprob(ngram ng, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL);
virtual double clprob(ngram ng, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL);
-// virtual double clprob(int* ng, int ngsize, double* bow=NULL, int* bol=NULL, ngram_state_t* maxsuffidx=NULL, char** maxsuffptr=NULL, unsigned int* statesize=NULL, bool* extendible=NULL, double* lastbow=NULL);
-
+
+ virtual const char *maxsuffptr(ngram ong, unsigned int* size=NULL);
+ virtual const char *cmaxsuffptr(ngram ong, unsigned int* size=NULL);
+ virtual ngram_state_t maxsuffidx(ngram ong, unsigned int* size=NULL);
+ virtual ngram_state_t cmaxsuffidx(ngram ong, unsigned int* size=NULL);
void *search(int lev,table_entry_pos_t offs,table_entry_pos_t n,int sz,int *w, LMT_ACTION action,char **found=(char **)NULL);
@@ -355,13 +357,6 @@ namespace irstlm {
int succscan(ngram& h,ngram& ng,LMT_ACTION action,int lev);
- virtual const char *maxsuffptr(ngram ong, unsigned int* size=NULL);
- virtual const char *cmaxsuffptr(ngram ong, unsigned int* size=NULL);
-// virtual const char *cmaxsuffptr(int* codes, int sz, unsigned int* size=NULL);
- virtual ngram_state_t maxsuffidx(ngram ong, unsigned int* size=NULL);
- virtual ngram_state_t cmaxsuffidx(ngram ong, unsigned int* size=NULL);
-// virtual ngram_state_t cmaxsuffidx(int* codes, int sz, unsigned int* size=NULL);
-
inline void putmem(char* ptr,int value,int offs,int size) {
MY_ASSERT(ptr!=NULL);
for (int i=0; i<size; i++)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/irstlm.git
More information about the debian-science-commits
mailing list