[irstlm] 93/126: multiple ngrams for each thread; yet to debug

[Giulio Paci]

This is an automated email from the git hooks/post-receive script.

giuliopaci-guest pushed a commit to annotated tag adaptiveLM.v0.1
in repository irstlm.

commit 8bf3fd99daa339ccd135672413524c7bae5fb8e4
Author: Nicola Bertoldi <bertoldi at fbk.eu>
Date:   Mon Sep 21 11:55:38 2015 +0200

    multiple ngrams for each thread; yet to debug
---
 src/test-multithreading.cpp | 77 ++++++++++++++++++++++++++++++++++++---------
 1 file changed, 62 insertions(+), 15 deletions(-)

diff --git a/src/test-multithreading.cpp b/src/test-multithreading.cpp
index 1b7842e..4015272 100644
--- a/src/test-multithreading.cpp
+++ b/src/test-multithreading.cpp
@@ -35,6 +35,16 @@
 #include <pthread.h>
 #include "thpool.h"
 
+#include <math.h>  //for ceil function
+
+struct task_array {      //basic task info to run task
+	void *ctx;
+	std::vector<ngram> *in;
+	std::vector<double> *out;
+	int start_pos;
+	int end_pos;
+};
+
 struct task {      //basic task info to run task
 	void *ctx;
 	std::vector<ngram> *in;
@@ -49,18 +59,29 @@ using namespace std;
 using namespace irstlm;
 
 
+
+
 static void *clprob_helper(void *argv){
 	task t=*(task *)argv;
 	(t.out)->at(t.pos) = ((lmContainer*) t.ctx)->clprob((t.in)->at(t.pos));
 	/*
-  std::cout << "clprob_helper()" << std::endl;
-	std::cout << "### (lmContainer*) t.ctx):|" << (void*)((lmContainer*) t.ctx) << "|" << std::endl;
-	std::cout << "### t.in:|" << (void*) t.in << "|" << std::endl;
-	std::cout << "### t.out:|" << (void*) t.out << "|" << std::endl;
-	std::cout << "### ng:|" << (t.in)->at(t.pos) << "|" << std::endl;
-	std::cout << "### Pr:|" << (t.out)->at(t.pos) << "|" << std::endl;
-*/
+	 std::cout << "clprob_helper()" << std::endl;
+	 std::cout << "### (lmContainer*) t.ctx):|" << (void*)((lmContainer*) t.ctx) << "|" << std::endl;
+	 std::cout << "### t.in:|" << (void*) t.in << "|" << std::endl;
+	 std::cout << "### t.out:|" << (void*) t.out << "|" << std::endl;
+	 std::cout << "### ng:|" << (t.in)->at(t.pos) << "|" << std::endl;
+	 std::cout << "### Pr:|" << (t.out)->at(t.pos) << "|" << std::endl;
+	 */
+	
+	return NULL;
+};
 
+static void *clprob_array_helper(void *argv){
+	task_array t=*(task_array *)argv;
+	for (int i=t.start_pos; i<t.end_pos; ++i){
+		(t.out)->at(i) = ((lmContainer*) t.ctx)->clprob((t.in)->at(i));
+	}
+	
 	return NULL;
 };
 
@@ -161,15 +182,16 @@ int main(int argc, char **argv)
 			Pr = lmt->clprob(ngram_vec.at(i));
 			prob_vec.at(i) = Pr;
 		}
-/*
- for (size_t i=0 ; i<ngram_vec_size; ++i){
-			std::cout << "prob_vec[" << i << "]=" << prob_vec[i] << " ng:|" << ngram_vec.at(i) << "|" << std::endl;
-		}
-*/		
+		/*
+		 for (size_t i=0 ; i<ngram_vec_size; ++i){
+		 std::cout << "prob_vec[" << i << "]=" << prob_vec[i] << " ng:|" << ngram_vec.at(i) << "|" << std::endl;
+		 }
+		 */		
 		
 		
     threadpool thpool=thpool_init(threads);
 		
+		/*
     int numtasks=ngram_vec_size;
     task *t=new task[numtasks];
 		for (size_t i=0 ; i<ngram_vec_size; ++i){
@@ -178,11 +200,36 @@ int main(int argc, char **argv)
 			t[i].in=&ngram_vec;
 			t[i].out=&thread_prob_vec;
 			t[i].pos=i;
-//			std::cout << "creating thread_task .... lmt:" << (void*) lmt << " i:" << i << " ng:|" << ngram_vec.at(i) << "|" << std::endl;
+			//			std::cout << "creating thread_task .... lmt:" << (void*) lmt << " i:" << i << " ng:|" << ngram_vec.at(i) << "|" << std::endl;
 			thpool_add_work(thpool, &clprob_helper, (void *)&t[i]);
 		}
 		//join all threads
 		thpool_wait(thpool);
+		*/
+		
+//		int step=10;
+//    int numtasks=ceil(ngram_vec_size/step);
+		
+		int numtasks=threads * 10;
+		int step=ceil(ngram_vec_size/numtasks);
+		
+    task_array *t=new task_array[numtasks];
+		int thread_i=0;
+		for (size_t j=0 ; j<ngram_vec_size; j+=step){
+			//prepare and assign tasks to threads
+			t[thread_i].ctx=lmt;
+			t[thread_i].in=&ngram_vec;
+			t[thread_i].out=&thread_prob_vec;
+			t[thread_i].start_pos=j;
+			t[thread_i].end_pos=(j+step<ngram_vec_size)?j+step:ngram_vec_size;
+			thpool_add_work(thpool, &clprob_array_helper, (void *)&t[thread_i]);
+			std::cout << "preparing thread " << thread_i << " start_pos:" << t[thread_i].start_pos<< " end_pos:" << t[thread_i].end_pos << std::endl;
+			++thread_i;
+		}
+  	std::cout << "ngram_vec_size:" << ngram_vec_size  << " threads:" << threads << " numtasks: " << numtasks << " step:" << step << " thread_i:" << thread_i << std::endl;
+		//join all threads
+		thpool_wait(thpool);
+		
 		
 		int errors=0;
 		for (size_t i=0 ; i<ngram_vec_size; ++i){
@@ -193,7 +240,7 @@ int main(int argc, char **argv)
 			}
 		}
 		
-  	std::cout << "There are " << errors << " errors" << std::endl;
-
+  	std::cout << "There are " << errors << " errors in " << ngram_vec_size << " ngram prob queries with " << threads << " threads" << std::endl;
+		
 	}
 }

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/irstlm.git