[irstlm] 88/126: added a program to test multi-threading
Giulio Paci
giuliopaci-guest at moszumanska.debian.org
Tue May 17 07:46:48 UTC 2016
This is an automated email from the git hooks/post-receive script.
giuliopaci-guest pushed a commit to annotated tag adaptiveLM.v0.1
in repository irstlm.
commit bb41d4bd624bf5dbd8bb2265bb03f07c19e74cb8
Author: Nicola Bertoldi <bertoldi at fbk.eu>
Date: Fri Sep 18 09:00:14 2015 +0200
added a program to test multi-threading
---
src/CMakeLists.txt | 8 +-
src/Makefile.am | 4 +-
src/test_multithreading.cpp | 201 ++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 210 insertions(+), 3 deletions(-)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index a23784e..4d55cea 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -6,7 +6,11 @@ ADD_DEFINITIONS("-D_LARGE_FILES")
ADD_DEFINITIONS("-D_FILE_OFFSET_BITS=64")
ADD_DEFINITIONS("-DMYCODESIZE=3")
ADD_DEFINITIONS("-DDEBUG")
-ADD_DEFINITIONS("-DTRACE_LEVEL=1")
+##ADD_DEFINITIONS("-DTRACE_LEVEL=1")
+
+if (TRACE_LEVEL)
+ ADD_DEFINITIONS("-DTRACE_LEVEL=${TRACE_LEVEL}")
+endif()
if (CXX0)
MESSAGE( STATUS "HAVE_CXX0=true; hence, variable HAVE_CXX0 is set" )
@@ -56,7 +60,7 @@ SET( LIB_IRSTLM_SRC
ADD_LIBRARY(irstlm STATIC ${LIB_IRSTLM_SRC})
LINK_DIRECTORIES (${LIBRARY_OUTPUT_PATH})
-FOREACH(CMD dict ngt tlm dtsel plsa cswa compile-lm interpolate-lm prune-lm quantize-lm score-lm)
+FOREACH(CMD dict ngt tlm dtsel plsa cswa compile-lm interpolate-lm prune-lm quantize-lm score-lm test_multithreading)
ADD_EXECUTABLE(${CMD} ${CMD}.cpp)
TARGET_LINK_LIBRARIES (${CMD} irstlm -lm -lz -lpthread)
diff --git a/src/Makefile.am b/src/Makefile.am
index 1f8a065..63d89de 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -69,7 +69,7 @@ libirstlm_la_LIBADD = $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB)
LDADD = -lirstlm -lpthread
DEPENDENCIES = libirstlm.la
-bin_PROGRAMS = dict ngt dtsel compile-lm interpolate-lm prune-lm quantize-lm prune-lm score-lm tlm plsa verify-caching cswa
+bin_PROGRAMS = dict ngt dtsel compile-lm interpolate-lm prune-lm quantize-lm prune-lm score-lm tlm plsa verify-caching cswa test_multithreading
dict_SOURCES = dict.cpp
dict_DEPENDENCIES = $(DEPENDENCIES)
@@ -95,3 +95,5 @@ verify_caching_SOURCES = verify-caching.cpp
verify_caching_DEPENDENCIES = $(DEPENDENCIES)
cswa_SOURCES = cswa.cpp
cswa_DEPENDENCIES = $(DEPENDENCIES)
+test_multithreading_SOURCES = test_multithreading.cpp
+test_multithreading_DEPENDENCIES = $(DEPENDENCIES)
diff --git a/src/test_multithreading.cpp b/src/test_multithreading.cpp
new file mode 100644
index 0000000..f7a39a5
--- /dev/null
+++ b/src/test_multithreading.cpp
@@ -0,0 +1,201 @@
+// $Id: test_multithreading.cpp 3677 2010-10-13 09:06:51Z bertoldi $
+
+/******************************************************************************
+ IrstLM: IRST Language Model Toolkit, compile LM
+ Copyright (C) 2006 Marcello Federico, ITC-irst Trento, Italy
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+ ******************************************************************************/
+
+
+#include <iostream>
+#include <string>
+#include <stdlib.h>
+#include <vector>
+#include "cmd.h"
+#include "util.h"
+#include "n_gram.h"
+#include "mdiadapt.h"
+#include "lmContainer.h"
+
+
+#include <pthread.h>
+#include "thpool.h"
+
+struct task { //basic task info to run task
+ void *ctx;
+ std::vector<ngram> *in;
+ std::vector<double> *out;
+ int pos;
+};
+
+
+
+/********************************/
+using namespace std;
+using namespace irstlm;
+
+
+static void *clprob_helper(void *argv){
+ task t=*(task *)argv;
+ (t.out)->at(t.pos) = ((lmContainer*) t.ctx)->clprob((t.in)->at(t.pos));
+ /*
+ std::cout << "clprob_helper()" << std::endl;
+ std::cout << "### (lmContainer*) t.ctx):|" << (void*)((lmContainer*) t.ctx) << "|" << std::endl;
+ std::cout << "### t.in:|" << (void*) t.in << "|" << std::endl;
+ std::cout << "### t.out:|" << (void*) t.out << "|" << std::endl;
+ std::cout << "### ng:|" << (t.in)->at(t.pos) << "|" << std::endl;
+ std::cout << "### Pr:|" << (t.out)->at(t.pos) << "|" << std::endl;
+*/
+
+ return NULL;
+};
+
+void print_help(int TypeFlag=0){
+ std::cerr << std::endl << "test_multithreading - test for multi-threading" << std::endl;
+ std::cerr << std::endl << "USAGE:" << std::endl;
+ std::cerr << " test_multithreading" << std::endl;
+ std::cerr << std::endl << "DESCRIPTION:" << std::endl;
+ std::cerr << std::endl << "OPTIONS:" << std::endl;
+
+ FullPrintParams(TypeFlag, 0, 1, stderr);
+}
+
+void usage(const char *msg = 0)
+{
+ if (msg) {
+ std::cerr << msg << std::endl;
+ }
+ if (!msg){
+ print_help();
+ }
+}
+
+int main(int argc, char **argv)
+{
+ bool help=false;
+ char *lmfile=NULL;
+ char *testfile=NULL;
+ int threads=1;
+
+ DeclareParams((char*)
+
+ "Help", CMDBOOLTYPE|CMDMSG, &help, "print this help",
+ "h", CMDBOOLTYPE|CMDMSG, &help, "print this help",
+
+ "lm", CMDSTRINGTYPE|CMDMSG, &lmfile, "config file",
+ "test", CMDSTRINGTYPE|CMDMSG, &testfile, "test file",
+ "thread", CMDINTTYPE|CMDMSG, &threads, "threads",
+
+ (char *)NULL
+ );
+
+ std::cout << "argc:" << argc << std::endl;
+/*
+ if (argc > 0){
+ usage();
+ exit_error(IRSTLM_NO_ERROR);
+ }
+ */
+
+ GetParams(&argc, &argv, (char*) NULL);
+
+ std::cout << "help:" << help << std::endl;
+ std::cout << "lmfile:" << lmfile << std::endl;
+
+ if (help){
+ usage();
+ exit_error(IRSTLM_NO_ERROR);
+ }
+
+ if (!lmfile) {
+ exit_error(IRSTLM_ERROR_DATA,"The LM file (-lm) is not specified");
+ }
+
+ lmContainer* lmt = lmContainer::CreateLanguageModel(lmfile);
+
+ lmt->load(lmfile);
+
+ if (testfile != NULL) {
+
+ std::fstream inptxt(testfile,std::ios::in);
+
+ std::vector<ngram> ngram_vec;
+ std::vector<double> prob_vec;
+ std::vector<double> thread_prob_vec;
+
+
+ ngram ng(lmt->getDict());
+
+ ng.dict->incflag(1);
+ /*
+ int bos=ng.dict->encode(ng.dict->BoS());
+ int eos=ng.dict->encode(ng.dict->EoS());
+ */
+ ng.dict->incflag(0);
+
+ while(inptxt >> ng) {
+ if (ng.size>=1) {
+ ngram_vec.push_back(ng);
+ }
+ }
+
+ size_t ngram_vec_size = ngram_vec.size();
+
+ prob_vec.resize(ngram_vec_size);
+ thread_prob_vec.resize(ngram_vec_size);
+
+ double Pr = -1000;
+ for (size_t i=0 ; i<ngram_vec_size; ++i){
+ Pr = lmt->clprob(ngram_vec.at(i));
+ prob_vec.at(i) = Pr;
+ }
+/*
+ for (size_t i=0 ; i<ngram_vec_size; ++i){
+ std::cout << "prob_vec[" << i << "]=" << prob_vec[i] << " ng:|" << ngram_vec.at(i) << "|" << std::endl;
+ }
+*/
+
+
+ threadpool thpool=thpool_init(threads);
+
+ int numtasks=ngram_vec_size;
+ task *t=new task[numtasks];
+ for (size_t i=0 ; i<ngram_vec_size; ++i){
+ //prepare and assign tasks to threads
+ t[i].ctx=lmt;
+ t[i].in=&ngram_vec;
+ t[i].out=&thread_prob_vec;
+ t[i].pos=i;
+// std::cout << "creating thread_task .... lmt:" << (void*) lmt << " i:" << i << " ng:|" << ngram_vec.at(i) << "|" << std::endl;
+ thpool_add_work(thpool, &clprob_helper, (void *)&t[i]);
+ }
+ //join all threads
+ thpool_wait(thpool);
+
+ int errors=0;
+ for (size_t i=0 ; i<ngram_vec_size; ++i){
+ if (thread_prob_vec[i] != prob_vec[i]){
+ std::cout << "thread_prob_vec[" << i << "]=" << thread_prob_vec[i];
+ std::cout << " ERROR" << std::endl;
+ ++errors;
+ }
+ }
+
+ std::cout << "There are " << errors << " errors" << std::endl;
+
+ }
+}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/irstlm.git
More information about the debian-science-commits
mailing list