[cg3] 01/03: Imported Upstream version 0.9.9~r10800
Tino Didriksen
tinodidriksen-guest at moszumanska.debian.org
Tue Jun 23 16:39:04 UTC 2015
This is an automated email from the git hooks/post-receive script.
tinodidriksen-guest pushed a commit to branch master
in repository cg3.
commit dedafe128e7a81e4a45d7e9700c405d308c1915d
Author: Tino Didriksen <mail at tinodidriksen.com>
Date: Tue Jun 23 16:38:37 2015 +0000
Imported Upstream version 0.9.9~r10800
---
ChangeLog | 47 ++++++-------
src/ApertiumApplicator.cpp | 22 +++---
src/Cohort.cpp | 78 ++++++++++++++++++++-
src/Cohort.hpp | 6 +-
src/CohortIterator.cpp | 4 ++
src/FSTApplicator.cpp | 10 +--
src/GrammarApplicator_reflow.cpp | 16 +++--
src/GrammarApplicator_runContextualTest.cpp | 3 +
src/GrammarApplicator_runGrammar.cpp | 22 +++---
src/GrammarApplicator_runRules.cpp | 23 +++----
src/NicelineApplicator.cpp | 10 +--
src/PlaintextApplicator.cpp | 8 +--
src/Reading.cpp | 102 ++++++++++++++++++++++------
src/Reading.hpp | 8 ++-
src/SingleWindow.cpp | 68 ++++++++++++++++++-
src/SingleWindow.hpp | 5 +-
src/Window.cpp | 6 +-
src/inlines.hpp | 44 +++++++++++-
src/libcg3.cpp | 8 +--
src/version.hpp | 2 +-
20 files changed, 373 insertions(+), 119 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 91d12a7..6a63470 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,26 @@
2015-06-23 tino
+ * [r10799] src/GrammarApplicator_reflow.cpp: More removed cohort
+ badness
+ * [r10798] src/inlines.hpp: Demote CR and NEL to just spaces
+ * [r10797] ChangeLog, src/ApertiumApplicator.cpp, src/Cohort.cpp,
+ src/Cohort.hpp, src/FSTApplicator.cpp,
+ src/GrammarApplicator_reflow.cpp,
+ src/GrammarApplicator_runGrammar.cpp,
+ src/GrammarApplicator_runRules.cpp, src/NicelineApplicator.cpp,
+ src/PlaintextApplicator.cpp, src/Reading.cpp, src/Reading.hpp,
+ src/SingleWindow.cpp, src/inlines.hpp, src/libcg3.cpp,
+ src/version.hpp: More pooling
+ * [r10796] src/CohortIterator.cpp,
+ src/GrammarApplicator_runContextualTest.cpp: Fix bad reaction
+ between removed cohorts and dependency
+ * [r10793] ChangeLog, scripts/profile-revisions.php,
+ src/Cohort.cpp, src/Cohort.hpp, src/Grammar.cpp, src/Grammar.hpp,
+ src/GrammarApplicator_matchSet.cpp,
+ src/GrammarApplicator_reflow.cpp,
+ src/GrammarApplicator_runRules.cpp, src/inlines.hpp,
+ src/stdafx.hpp, src/version.hpp: Instead of using upwards 128KB
+ per cohort possible_sets cache, use 2KB
* [r10791] ChangeLog, src/Grammar.hpp, src/Reading.cpp,
src/flat_unordered_map.hpp, src/flat_unordered_set.hpp,
src/sorted_vector.hpp, src/version.hpp: Let's ship a complete
@@ -2301,30 +2322,4 @@
* [r8708] src/GrammarApplicator_runGrammar.cpp: ; in input does not
mean end of line...
- * [r8707] scripts/cg3-autobin.pl,
- src/GrammarApplicator_runGrammar.cpp, src/inlines.h,
- src/version.h: ; in input does not mean end of line...
-
-2012-11-21 tino
-
- * [r8702] scripts/cg3-autobin.pl,
- src/GrammarApplicator_runGrammar.cpp, src/inlines.h,
- src/version.h: # in input does not mean comment...
-
-2012-11-15 tino
-
- * [r8691] src/options.h: Swapped text on -I -O
-
-2012-10-30 tino
-
- * [r8665] scripts/cg3-autobin.pl, src/GrammarApplicator_reflow.cpp,
- src/version.h: Max cohort for reflowDep should only apply when
- dep-delimit is in effect.
-
-2012-10-25 tino
-
- * [r8661] scripts/cg3-autobin.pl,
- src/GrammarApplicator_matchSet.cpp,
- src/GrammarApplicator_runRules.cpp, src/version.h: Context tests
- looking at Deleted and Delayed need to bypass the cache.
diff --git a/src/ApertiumApplicator.cpp b/src/ApertiumApplicator.cpp
index 7d11ee4..314fc38 100644
--- a/src/ApertiumApplicator.cpp
+++ b/src/ApertiumApplicator.cpp
@@ -246,11 +246,11 @@ void ApertiumApplicator::runGrammarOnText(istream& input, UFILE *output) {
cSWindow = gWindow->allocAppendSingleWindow();
// Create 0th Cohort which serves as the beginning of sentence
- cCohort = new Cohort(cSWindow);
+ cCohort = alloc_cohort(cSWindow);
cCohort->global_number = 0;
cCohort->wordform = tag_begin;
- cReading = new Reading(cCohort);
+ cReading = alloc_reading(cCohort);
cReading->baseform = begintag;
insert_if_exists(cReading->parent->possible_sets, grammar->sets_any);
addTagToReading(*cReading, begintag);
@@ -275,7 +275,7 @@ void ApertiumApplicator::runGrammarOnText(istream& input, UFILE *output) {
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -284,7 +284,7 @@ void ApertiumApplicator::runGrammarOnText(istream& input, UFILE *output) {
resetIndexes();
}
}
- cCohort = new Cohort(cSWindow);
+ cCohort = alloc_cohort(cSWindow);
cCohort->global_number = gWindow->cohort_counter++;
// Read in the word form
@@ -323,7 +323,7 @@ void ApertiumApplicator::runGrammarOnText(istream& input, UFILE *output) {
// Gobble up all <tags> until the first / or $ and stuff them in the static reading
if (inchar == '<') {
//u_fprintf(ux_stderr, "Static reading\n");
- cCohort->wread.reset(new Reading(cCohort));
+ cCohort->wread = alloc_reading(cCohort);
UString tag;
do {
inchar = u_fgetc_wrapper(input);
@@ -362,7 +362,7 @@ void ApertiumApplicator::runGrammarOnText(istream& input, UFILE *output) {
if (inchar == '$') {
// Add the final reading of the cohort
- cReading = new Reading(cCohort);
+ cReading = alloc_reading(cCohort);
insert_if_exists(cReading->parent->possible_sets, grammar->sets_any);
@@ -383,7 +383,7 @@ void ApertiumApplicator::runGrammarOnText(istream& input, UFILE *output) {
if (inchar == '/') { // Reached end of reading
Reading *cReading = 0;
- cReading = new Reading(cCohort);
+ cReading = alloc_reading(cCohort);
addTagToReading(*cReading, cCohort->wordform);
@@ -435,7 +435,7 @@ void ApertiumApplicator::runGrammarOnText(istream& input, UFILE *output) {
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -446,7 +446,7 @@ void ApertiumApplicator::runGrammarOnText(istream& input, UFILE *output) {
while (!gWindow->previous.empty()) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
@@ -668,7 +668,7 @@ void ApertiumApplicator::testPR(UFILE *output) {
};
for (size_t i = 0 ; i<6 ; ++i) {
UString text(texts[i].begin(), texts[i].end());
- Reading *reading = new Reading(0);
+ Reading *reading = alloc_reading(0);
processReading(reading, text);
if (grammar->sub_readings_ltr && reading->next) {
reading = reverse(reading);
@@ -928,7 +928,7 @@ void ApertiumApplicator::mergeMappings(Cohort& cohort) {
std::map<uint32_t, ReadingList>::iterator miter;
for (miter = mlist.begin() ; miter != mlist.end() ; miter++) {
ReadingList clist = miter->second;
- Reading *nr = new Reading(*(clist.front()));
+ Reading *nr = alloc_reading(*(clist.front()));
// no merging of mapping tags
order.push_back(nr);
}
diff --git a/src/Cohort.cpp b/src/Cohort.cpp
index eac90ff..95e0a50 100644
--- a/src/Cohort.cpp
+++ b/src/Cohort.cpp
@@ -30,6 +30,27 @@
namespace CG3 {
+CohortVector pool_cohorts;
+pool_cleaner<CohortVector> cleaner_cohorts(pool_cohorts);
+
+Cohort *alloc_cohort(SingleWindow *p) {
+ Cohort *c = pool_get(pool_cohorts);
+ if (c == 0) {
+ c = new Cohort(p);
+ }
+ else {
+ c->parent = p;
+ }
+ return c;
+}
+
+void free_cohort(Cohort *c) {
+ if (c == 0) {
+ return;
+ }
+ pool_put(pool_cohorts, c);
+}
+
Cohort::Cohort(SingleWindow *p) :
type(0),
global_number(0),
@@ -41,7 +62,8 @@ is_pleft(0),
is_pright(0),
parent(p),
prev(0),
-next(0)
+next(0),
+wread(0)
{
#ifdef CG_TRACE_OBJECTS
std::cerr << "OBJECT: " << __PRETTY_FUNCTION__ << std::endl;
@@ -62,7 +84,9 @@ Cohort::~Cohort() {
foreach (ReadingList, delayed, iter3, iter3_end) {
delete (*iter3);
}
- foreach (CohortVector, removed, iter, iter_end) {
+ delete wread;
+
+ foreach(CohortVector, removed, iter, iter_end) {
delete (*iter);
}
if (parent) {
@@ -72,6 +96,54 @@ Cohort::~Cohort() {
detach();
}
+void Cohort::clear() {
+ if (parent) {
+ parent->parent->cohort_map.erase(global_number);
+ parent->parent->dep_window.erase(global_number);
+ }
+ detach();
+
+ type = 0;
+ global_number = 0;
+ local_number = 0;
+ wordform = 0;
+ dep_self = 0;
+ dep_parent = std::numeric_limits<uint32_t>::max();
+ is_pleft = 0;
+ is_pright = 0;
+ parent = 0;
+
+ text.clear();
+ num_max.clear();
+ num_min.clear();
+ dep_children.clear();
+ possible_sets.clear();
+ relations.clear();
+ relations_input.clear();
+
+ foreach(ReadingList, readings, iter1, iter1_end) {
+ free_reading(*iter1);
+ }
+ foreach(ReadingList, deleted, iter2, iter2_end) {
+ free_reading(*iter2);
+ }
+ foreach(ReadingList, delayed, iter3, iter3_end) {
+ free_reading(*iter3);
+ }
+ free_reading(wread);
+
+ readings.clear();
+ deleted.clear();
+ delayed.clear();
+ wread = 0;
+
+ foreach(CohortVector, removed, iter, iter_end) {
+ free_cohort(*iter);
+ }
+ removed.clear();
+ assert(enclosed.empty() && "Enclosed was not empty!");
+}
+
void Cohort::detach() {
if (prev) {
prev->next = next;
@@ -99,7 +171,7 @@ void Cohort::appendReading(Reading *read) {
}
Reading* Cohort::allocateAppendReading() {
- Reading *read = new Reading(this);
+ Reading *read = alloc_reading(this);
readings.push_back(read);
if (read->number == 0) {
read->number = (uint32_t)readings.size() * 1000 + 1000;
diff --git a/src/Cohort.hpp b/src/Cohort.hpp
index f6f4219..6bb520c 100644
--- a/src/Cohort.hpp
+++ b/src/Cohort.hpp
@@ -55,7 +55,7 @@ namespace CG3 {
SingleWindow *parent;
UString text;
Cohort *prev, *next;
- boost::scoped_ptr<Reading> wread;
+ Reading *wread;
ReadingList readings;
ReadingList deleted;
ReadingList delayed;
@@ -75,6 +75,7 @@ namespace CG3 {
Cohort(SingleWindow *p);
~Cohort();
+ void clear();
void addChild(uint32_t child);
void remChild(uint32_t child);
@@ -96,6 +97,9 @@ namespace CG3 {
typedef sorted_vector<Cohort*, compare_Cohort> CohortSet;
typedef stdext::hash_map<uint32_t, CohortSet> uint32ToCohortsMap;
+
+ Cohort *alloc_cohort(SingleWindow *p);
+ void free_cohort(Cohort *c);
}
#endif
diff --git a/src/CohortIterator.cpp b/src/CohortIterator.cpp
index 60f8a84..f1b005e 100644
--- a/src/CohortIterator.cpp
+++ b/src/CohortIterator.cpp
@@ -112,6 +112,10 @@ DepParentIter& DepParentIter::operator++() {
std::map<uint32_t,Cohort*>::iterator it = m_cohort->parent->parent->cohort_map.find(m_cohort->dep_parent);
if (it != m_cohort->parent->parent->cohort_map.end()) {
Cohort *cohort = it->second;
+ if (cohort->type & CT_REMOVED) {
+ m_cohort = 0;
+ return *this;
+ }
if (m_seen.find(cohort) == m_seen.end()) {
m_seen.insert(m_cohort);
if (cohort->parent == m_cohort->parent || (m_test->pos & POS_SPAN_BOTH) || m_span) {
diff --git a/src/FSTApplicator.cpp b/src/FSTApplicator.cpp
index 67c145c..515a92e 100644
--- a/src/FSTApplicator.cpp
+++ b/src/FSTApplicator.cpp
@@ -158,7 +158,7 @@ gotaline:
++numWindows;
did_soft_lookback = false;
}
- cCohort = new Cohort(cSWindow);
+ cCohort = alloc_cohort(cSWindow);
cCohort->global_number = gWindow->cohort_counter++;
cCohort->wordform = addTag(tag);
lCohort = cCohort;
@@ -167,7 +167,7 @@ gotaline:
++space;
while (space && (space[0] != '+' || space[1] != '?' || space[2] != 0)) {
- cReading = new Reading(cCohort);
+ cReading = alloc_reading(cCohort);
insert_if_exists(cReading->parent->possible_sets, grammar->sets_any);
addTagToReading(*cReading, cCohort->wordform);
@@ -360,7 +360,7 @@ istext:
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -408,7 +408,7 @@ istext:
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -419,7 +419,7 @@ istext:
while (!gWindow->previous.empty()) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
diff --git a/src/GrammarApplicator_reflow.cpp b/src/GrammarApplicator_reflow.cpp
index 7e98013..2ceeaa4 100644
--- a/src/GrammarApplicator_reflow.cpp
+++ b/src/GrammarApplicator_reflow.cpp
@@ -220,7 +220,11 @@ void GrammarApplicator::reflowDependencyWindow(uint32_t max) {
gWindow->cohort_map[0] = gWindow->current->cohorts[0];
}
else if (gWindow->cohort_map.find(0) == gWindow->cohort_map.end()) {
- Cohort *tmp = gWindow->cohort_map.begin()->second->parent->cohorts[0];
+ Cohort *tmp = gWindow->current->cohorts[0];
+ Cohort *c = gWindow->cohort_map.begin()->second;
+ if (c->parent) {
+ tmp = c->parent->cohorts[0];
+ }
gWindow->cohort_map[0] = tmp;
}
@@ -592,7 +596,7 @@ void GrammarApplicator::splitMappings(TagList& mappings, Cohort& cohort, Reading
if (found) {
continue;
}
- Reading *nr = new Reading(reading);
+ Reading *nr = alloc_reading(reading);
nr->mapped = mapped;
nr->number = reading.number - i--;
uint32_t mp = addTagToReading(*nr, *ttag);
@@ -691,7 +695,7 @@ void GrammarApplicator::mergeReadings(ReadingList& readings) {
for (BOOST_AUTO(miter, mlist.begin()) ; miter != mlist.end() ; miter++) {
ReadingList clist = miter->second;
- Reading *nr = new Reading(*(clist.front()));
+ Reading *nr = alloc_reading(*(clist.front()));
if (nr->mapping) {
erase(nr->tags_list, nr->mapping->hash);
}
@@ -699,7 +703,7 @@ void GrammarApplicator::mergeReadings(ReadingList& readings) {
if ((*iter1)->mapping && std::find(nr->tags_list.begin(), nr->tags_list.end(), (*iter1)->mapping->hash) == nr->tags_list.end()) {
nr->tags_list.push_back((*iter1)->mapping->hash);
}
- delete (*iter1);
+ free_reading(*iter1);
}
order.push_back(nr);
}
@@ -746,11 +750,11 @@ Cohort *GrammarApplicator::delimitAt(SingleWindow& current, Cohort *cohort) {
nwin->has_enclosures = current.has_enclosures;
current.parent->cohort_counter++;
- Cohort *cCohort = new Cohort(nwin);
+ Cohort *cCohort = alloc_cohort(nwin);
cCohort->global_number = 0;
cCohort->wordform = tag_begin;
- Reading *cReading = new Reading(cCohort);
+ Reading *cReading = alloc_reading(cCohort);
cReading->baseform = begintag;
insert_if_exists(cReading->parent->possible_sets, grammar->sets_any);
addTagToReading(*cReading, begintag);
diff --git a/src/GrammarApplicator_runContextualTest.cpp b/src/GrammarApplicator_runContextualTest.cpp
index a5cc420..b96ecb1 100644
--- a/src/GrammarApplicator_runContextualTest.cpp
+++ b/src/GrammarApplicator_runContextualTest.cpp
@@ -581,6 +581,9 @@ Cohort *GrammarApplicator::runDependencyTest(SingleWindow *sWindow, Cohort *curr
continue;
}
Cohort *cohort = sWindow->parent->cohort_map.find(*dter)->second;
+ if (cohort->type & CT_REMOVED) {
+ continue;
+ }
bool good = true;
if (current->parent != cohort->parent) {
if ((!(test->pos & (POS_SPAN_BOTH|POS_SPAN_LEFT))) && cohort->parent->number < current->parent->number) {
diff --git a/src/GrammarApplicator_runGrammar.cpp b/src/GrammarApplicator_runGrammar.cpp
index 033011e..49dca3e 100644
--- a/src/GrammarApplicator_runGrammar.cpp
+++ b/src/GrammarApplicator_runGrammar.cpp
@@ -30,11 +30,11 @@
namespace CG3 {
void GrammarApplicator::initEmptySingleWindow(SingleWindow *cSWindow) {
- Cohort *cCohort = new Cohort(cSWindow);
+ Cohort *cCohort = alloc_cohort(cSWindow);
cCohort->global_number = 0;
cCohort->wordform = tag_begin;
- Reading *cReading = new Reading(cCohort);
+ Reading *cReading = alloc_reading(cCohort);
cReading->baseform = begintag;
insert_if_exists(cReading->parent->possible_sets, grammar->sets_any);
addTagToReading(*cReading, begintag);
@@ -45,7 +45,7 @@ void GrammarApplicator::initEmptySingleWindow(SingleWindow *cSWindow) {
}
Reading *GrammarApplicator::initEmptyCohort(Cohort& cCohort) {
- Reading *cReading = new Reading(&cCohort);
+ Reading *cReading = alloc_reading(&cCohort);
if (allow_magic_readings) {
cReading->baseform = makeBaseFromWord(cCohort.wordform)->hash;
}
@@ -252,7 +252,7 @@ gotaline:
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -265,7 +265,7 @@ gotaline:
u_fflush(ux_stderr);
}
}
- cCohort = new Cohort(cSWindow);
+ cCohort = alloc_cohort(cSWindow);
cCohort->global_number = gWindow->cohort_counter++;
cCohort->wordform = addTag(&cleaned[0]);
lCohort = cCohort;
@@ -275,7 +275,7 @@ gotaline:
space += 2;
if (space[0]) {
- cCohort->wread.reset(new Reading(cCohort));
+ cCohort->wread = alloc_reading(cCohort);
addTagToReading(*cCohort->wread, cCohort->wordform);
while (space[0]) {
SKIPWS(space, 0, 0, true);
@@ -312,7 +312,7 @@ gotaline:
indents.back().second->next = cReading;
}
else {
- cReading = new Reading(cCohort);
+ cReading = alloc_reading(cCohort);
}
insert_if_exists(cReading->parent->possible_sets, grammar->sets_any);
addTagToReading(*cReading, cCohort->wordform);
@@ -437,7 +437,7 @@ istext:
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -454,7 +454,7 @@ istext:
while (!gWindow->previous.empty()) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
u_fprintf(output, "%S", &line[0]);
@@ -614,7 +614,7 @@ istext:
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -629,7 +629,7 @@ istext:
while (!gWindow->previous.empty()) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
diff --git a/src/GrammarApplicator_runRules.cpp b/src/GrammarApplicator_runRules.cpp
index 94ddcbe..273e583 100644
--- a/src/GrammarApplicator_runRules.cpp
+++ b/src/GrammarApplicator_runRules.cpp
@@ -88,7 +88,10 @@ bool GrammarApplicator::updateValidRules(const uint32IntervalVector& rules, uint
void GrammarApplicator::indexSingleWindow(SingleWindow& current) {
current.valid_rules.clear();
- current.rule_to_cohorts.clear();
+ current.rule_to_cohorts.resize(grammar->rule_by_number.size());
+ boost_foreach(CohortSet& cs, current.rule_to_cohorts) {
+ cs.clear();
+ }
foreach (CohortVector, current.cohorts, iter, iter_end) {
Cohort *c = *iter;
@@ -293,11 +296,7 @@ uint32_t GrammarApplicator::runRulesOnSingleWindow(SingleWindow& current, const
const Set& set = *(grammar->sets_list[rule.target]);
grammar->lines = rule.line;
- uint32ToCohortsMap::iterator csit = current.rule_to_cohorts.find(rule.number);
- if (csit == current.rule_to_cohorts.end()) {
- continue;
- }
- CohortSet *cohortset = &csit->second;
+ CohortSet *cohortset = ¤t.rule_to_cohorts[rule.number];
if (debug_level > 1) {
std::cerr << "DEBUG: " << cohortset->size() << "/" << current.cohorts.size() << " = " << double(cohortset->size())/double(current.cohorts.size()) << std::endl;
}
@@ -737,7 +736,7 @@ uint32_t GrammarApplicator::runRulesOnSingleWindow(SingleWindow& current, const
intersects = current.valid_rules.intersect(rules);
iter_rules = intersects.find(rule.number);
iter_rules_end = intersects.end();
- cohortset = ¤t.rule_to_cohorts.find(rule.number)->second;
+ cohortset = ¤t.rule_to_cohorts[rule.number];
rocit = cohortset->find(cohort);
++rocit;
break;
@@ -770,7 +769,7 @@ uint32_t GrammarApplicator::runRulesOnSingleWindow(SingleWindow& current, const
}
}
index_ruleCohort_no.clear();
- cohortset = ¤t.rule_to_cohorts.find(rule.number)->second;
+ cohortset = ¤t.rule_to_cohorts[rule.number];
rocit = cohortset->find(cohort);
++rocit;
}
@@ -781,7 +780,7 @@ uint32_t GrammarApplicator::runRulesOnSingleWindow(SingleWindow& current, const
reading.hit_by.push_back(rule.number);
index_ruleCohort_no.clear();
- Cohort *cCohort = new Cohort(¤t);
+ Cohort *cCohort = alloc_cohort(¤t);
cCohort->global_number = gWindow->cohort_counter++;
Tag *wf = 0;
@@ -802,7 +801,7 @@ uint32_t GrammarApplicator::runRulesOnSingleWindow(SingleWindow& current, const
}
foreach (std::vector<TagList>, readings, rit, rit_end) {
- Reading *cReading = new Reading(cCohort);
+ Reading *cReading = alloc_reading(cCohort);
++numReadings;
insert_if_exists(cReading->parent->possible_sets, grammar->sets_any);
cReading->hit_by.push_back(rule.number);
@@ -864,7 +863,7 @@ uint32_t GrammarApplicator::runRulesOnSingleWindow(SingleWindow& current, const
indexSingleWindow(current);
readings_changed = true;
- cohortset = ¤t.rule_to_cohorts.find(rule.number)->second;
+ cohortset = ¤t.rule_to_cohorts[rule.number];
rocit = cohortset->find(cohort);
++rocit;
break;
@@ -1075,7 +1074,7 @@ uint32_t GrammarApplicator::runRulesOnSingleWindow(SingleWindow& current, const
}
foreach (std::vector<TagList>, readings, rit, rit_end) {
- Reading *cReading = new Reading(cohort);
+ Reading *cReading = alloc_reading(cohort);
++numReadings;
insert_if_exists(cReading->parent->possible_sets, grammar->sets_any);
addTagToReading(*cReading, cohort->wordform);
diff --git a/src/NicelineApplicator.cpp b/src/NicelineApplicator.cpp
index faf24d3..f2c041f 100644
--- a/src/NicelineApplicator.cpp
+++ b/src/NicelineApplicator.cpp
@@ -203,7 +203,7 @@ gotaline:
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -224,7 +224,7 @@ gotaline:
tag += '>';
tag += '"';
- cCohort = new Cohort(cSWindow);
+ cCohort = alloc_cohort(cSWindow);
cCohort->global_number = gWindow->cohort_counter++;
cCohort->wordform = addTag(tag);
lCohort = cCohort;
@@ -232,7 +232,7 @@ gotaline:
++space;
while (space && space[0]) {
- cReading = new Reading(cCohort);
+ cReading = alloc_reading(cCohort);
insert_if_exists(cReading->parent->possible_sets, grammar->sets_any);
UChar *base = space;
@@ -339,7 +339,7 @@ istext:
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -350,7 +350,7 @@ istext:
while (!gWindow->previous.empty()) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
diff --git a/src/PlaintextApplicator.cpp b/src/PlaintextApplicator.cpp
index 6db0d8a..6481a30 100644
--- a/src/PlaintextApplicator.cpp
+++ b/src/PlaintextApplicator.cpp
@@ -193,7 +193,7 @@ gotaline:
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -256,7 +256,7 @@ gotaline:
}
}
- cCohort = new Cohort(cSWindow);
+ cCohort = alloc_cohort(cSWindow);
cCohort->global_number = gWindow->cohort_counter++;
tag.clear();
tag += '"';
@@ -330,7 +330,7 @@ gotaline:
while (!gWindow->previous.empty() && gWindow->previous.size() > num_windows) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
gWindow->shuffleWindowsDown();
@@ -341,7 +341,7 @@ gotaline:
while (!gWindow->previous.empty()) {
SingleWindow *tmp = gWindow->previous.front();
printSingleWindow(tmp, output);
- delete tmp;
+ free_swindow(tmp);
gWindow->previous.pop_front();
}
diff --git a/src/Reading.cpp b/src/Reading.cpp
index 3fdc143..f3876cf 100644
--- a/src/Reading.cpp
+++ b/src/Reading.cpp
@@ -24,23 +24,60 @@
namespace CG3 {
-Reading::Reading() :
-mapped(false),
-deleted(false),
-noprint(false),
-matched_target(false),
-matched_tests(false),
-baseform(0),
-hash(0),
-hash_plain(0),
-number(0),
-mapping(0),
-parent(0),
-next(0)
-{
- #ifdef CG_TRACE_OBJECTS
- std::cerr << "OBJECT: " << __PRETTY_FUNCTION__ << std::endl;
- #endif
+ReadingList pool_readings;
+pool_cleaner<ReadingList> cleaner_readings(pool_readings);
+
+Reading *alloc_reading(Cohort *p) {
+ Reading *r = pool_get(pool_readings);
+ if (r == 0) {
+ r = new Reading(p);
+ }
+ else {
+ r->number = p ? (p->readings.size() * 1000 + 1000) : 0;
+ r->parent = p;
+ }
+ return r;
+}
+
+Reading *alloc_reading(const Reading& o) {
+ Reading *r = pool_get(pool_readings);
+ if (r == 0) {
+ r = new Reading(o);
+ }
+ else {
+ r->mapped = o.mapped;
+ r->deleted = o.deleted;
+ r->noprint = o.noprint;
+ r->matched_target = false;
+ r->matched_tests = false;
+ r->baseform = o.baseform;
+ r->hash = o.hash;
+ r->hash_plain = o.hash_plain;
+ r->number = o.number + 100;
+ r->tags_bloom = o.tags_bloom;
+ r->tags_plain_bloom = o.tags_plain_bloom;
+ r->tags_textual_bloom = o.tags_textual_bloom;
+ r->mapping = o.mapping;
+ r->parent = o.parent;
+ r->next = o.next;
+ r->hit_by = o.hit_by;
+ r->tags_list = o.tags_list;
+ r->tags = o.tags;
+ r->tags_plain = o.tags_plain;
+ r->tags_textual = o.tags_textual;
+ r->tags_numerical = o.tags_numerical;
+ if (r->next) {
+ r->next = alloc_reading(*r->next);
+ }
+ }
+ return r;
+}
+
+void free_reading(Reading *r) {
+ if (r == 0) {
+ return;
+ }
+ pool_put(pool_readings, r);
}
Reading::Reading(Cohort *p) :
@@ -52,7 +89,7 @@ matched_tests(false),
baseform(0),
hash(0),
hash_plain(0),
-number(p->readings.size() * 1000 + 1000),
+number(p ? (p->readings.size() * 1000 + 1000) : 0),
mapping(0),
parent(p),
next(0)
@@ -103,12 +140,37 @@ Reading::~Reading() {
next = 0;
}
+void Reading::clear() {
+ mapped = false;
+ deleted = false;
+ noprint = false;
+ matched_target = false;
+ matched_tests = false;
+ baseform = 0;
+ hash = 0;
+ hash_plain = 0;
+ number = 0;
+ tags_bloom.clear();
+ tags_plain_bloom.clear();
+ tags_textual_bloom.clear();
+ mapping = 0;
+ parent = 0;
+ free_reading(next);
+ next = 0;
+ hit_by.clear();
+ tags_list.clear();
+ tags.clear();
+ tags_plain.clear();
+ tags_textual.clear();
+ tags_numerical.clear();
+}
+
Reading *Reading::allocateReading(Cohort *p) {
- return new Reading(p);
+ return alloc_reading(p);
}
Reading *Reading::allocateReading(const Reading& r) {
- return new Reading(r);
+ return alloc_reading(r);
}
uint32_t Reading::rehash() {
diff --git a/src/Reading.hpp b/src/Reading.hpp
index 0d5cc99..5a27221 100644
--- a/src/Reading.hpp
+++ b/src/Reading.hpp
@@ -60,10 +60,10 @@ namespace CG3 {
typedef bc::flat_map<uint32_t,Tag*> tags_numerical_t;
tags_numerical_t tags_numerical;
- Reading();
- Reading(Cohort *p);
+ Reading(Cohort *p = 0);
Reading(const Reading& r);
~Reading();
+ void clear();
Reading *allocateReading(Cohort *p);
Reading *allocateReading(const Reading& r);
@@ -71,6 +71,10 @@ namespace CG3 {
uint32_t rehash();
static bool cmp_number(Reading *a, Reading *b);
};
+
+ Reading *alloc_reading(Cohort *p = 0);
+ Reading *alloc_reading(const Reading& r);
+ void free_reading(Reading *r);
}
#endif
diff --git a/src/SingleWindow.cpp b/src/SingleWindow.cpp
index 3063ce3..dc5ffb5 100644
--- a/src/SingleWindow.cpp
+++ b/src/SingleWindow.cpp
@@ -25,6 +25,27 @@
namespace CG3 {
+std::vector<SingleWindow*> pool_swindows;
+pool_cleaner< std::vector<SingleWindow*> > cleaner_swindows(pool_swindows);
+
+SingleWindow *alloc_swindow(Window *p) {
+ SingleWindow *s = pool_get(pool_swindows);
+ if (s == 0) {
+ s = new SingleWindow(p);
+ }
+ else {
+ s->parent = p;
+ }
+ return s;
+}
+
+void free_swindow(SingleWindow *s) {
+ if (s == 0) {
+ return;
+ }
+ pool_put(pool_swindows, s);
+}
+
SingleWindow::SingleWindow(Window *p) :
number(0),
has_enclosures(false),
@@ -54,7 +75,7 @@ SingleWindow::~SingleWindow() {
}
foreach (CohortVector, cohorts, iter, iter_end) {
- delete (*iter);
+ delete *iter;
}
if (next && previous) {
next->previous = previous;
@@ -70,6 +91,51 @@ SingleWindow::~SingleWindow() {
}
}
+void SingleWindow::clear() {
+ if (cohorts.size() > 1) {
+ for (uint32FlatHashMap::iterator iter = parent->relation_map.begin(); iter != parent->relation_map.end();) {
+ if (iter->second <= cohorts.back()->global_number) {
+ iter = parent->relation_map.erase(iter);
+ }
+ else {
+ ++iter;
+ }
+ }
+ }
+
+ foreach(CohortVector, cohorts, iter, iter_end) {
+ free_cohort(*iter);
+ }
+ if (next && previous) {
+ next->previous = previous;
+ previous->next = next;
+ }
+ else {
+ if (next) {
+ next->previous = 0;
+ }
+ if (previous) {
+ previous->next = 0;
+ }
+ }
+
+ number = 0;
+ has_enclosures = false;
+ next = 0;
+ previous = 0;
+ parent = 0;
+ text.clear();
+ cohorts.clear();
+ valid_rules.clear();
+ hit_external.clear();
+ boost_foreach (CohortSet& cs, rule_to_cohorts) {
+ cs.clear();
+ }
+ variables_set.clear();
+ variables_rem.clear();
+ variables_output.clear();
+}
+
void SingleWindow::appendCohort(Cohort *cohort) {
cohort->local_number = (uint32_t)cohorts.size();
cohort->parent = this;
diff --git a/src/SingleWindow.hpp b/src/SingleWindow.hpp
index 23da79a..e391072 100644
--- a/src/SingleWindow.hpp
+++ b/src/SingleWindow.hpp
@@ -42,17 +42,20 @@ namespace CG3 {
CohortVector cohorts;
uint32IntervalVector valid_rules;
uint32SortedVector hit_external;
- uint32ToCohortsMap rule_to_cohorts;
+ std::vector<CohortSet> rule_to_cohorts;
uint32FlatHashMap variables_set;
uint32FlatHashSet variables_rem;
uint32SortedVector variables_output;
SingleWindow(Window *p);
~SingleWindow();
+ void clear();
void appendCohort(Cohort *cohort);
};
+ SingleWindow *alloc_swindow(Window *p);
+ void free_swindow(SingleWindow *s);
}
#endif
diff --git a/src/Window.cpp b/src/Window.cpp
index ca8e08f..9c12fd0 100644
--- a/src/Window.cpp
+++ b/src/Window.cpp
@@ -50,14 +50,14 @@ Window::~Window() {
}
SingleWindow *Window::allocSingleWindow() {
- SingleWindow *swindow = new SingleWindow(this);
+ SingleWindow *swindow = alloc_swindow(this);
window_counter++;
swindow->number = window_counter;
return swindow;
}
SingleWindow *Window::allocPushSingleWindow() {
- SingleWindow *swindow = new SingleWindow(this);
+ SingleWindow *swindow = alloc_swindow(this);
window_counter++;
swindow->number = window_counter;
if (!next.empty()) {
@@ -73,7 +73,7 @@ SingleWindow *Window::allocPushSingleWindow() {
}
SingleWindow *Window::allocAppendSingleWindow() {
- SingleWindow *swindow = new SingleWindow(this);
+ SingleWindow *swindow = alloc_swindow(this);
window_counter++;
swindow->number = window_counter;
if (!next.empty()) {
diff --git a/src/inlines.hpp b/src/inlines.hpp
index 5e09bee..ae675e2 100644
--- a/src/inlines.hpp
+++ b/src/inlines.hpp
@@ -214,9 +214,6 @@ inline bool ISNL(const UChar c) {
return (
c == 0x2028L // Unicode Line Seperator
|| c == 0x2029L // Unicode Paragraph Seperator
- || c == 0x0085L // EBCDIC NEL
- || c == 0x0D0AL // Carriage Return + Line Feed
- || c == 0x000DL // Carriage Return
|| c == 0x000CL // Form Feed
|| c == 0x000BL // Vertical Tab
|| c == 0x000AL // ASCII \n
@@ -587,6 +584,47 @@ void pool_put(Pool& pool, Var& var) {
var.swap(pool.back());
}
+template<typename Pool, typename Var>
+void pool_get(Pool& pool, Var *& var) {
+ var = 0;
+ if (!pool.empty()) {
+ var = pool.back();
+ pool.pop_back();
+ }
+}
+
+template<typename Pool>
+typename Pool::value_type pool_get(Pool& pool) {
+ typename Pool::value_type var = 0;
+ if (!pool.empty()) {
+ var = pool.back();
+ pool.pop_back();
+ }
+ return var;
+}
+
+template<typename Pool, typename Var>
+void pool_put(Pool& pool, Var *var) {
+ var->clear();
+ pool.push_back(var);
+}
+
+template<typename Pool>
+struct pool_cleaner {
+ Pool& pool;
+
+ pool_cleaner(Pool& pool) :
+ pool(pool)
+ {
+ }
+
+ ~pool_cleaner() {
+ for (size_t i = 0; i < pool.size(); ++i) {
+ delete pool[i];
+ }
+ }
+};
+
}
#endif
diff --git a/src/libcg3.cpp b/src/libcg3.cpp
index 21006ec..f714508 100644
--- a/src/libcg3.cpp
+++ b/src/libcg3.cpp
@@ -195,7 +195,7 @@ inline Tag *_tag_copy(GrammarApplicator *from, GrammarApplicator *to, uint32_t h
}
inline Reading *_reading_copy(Cohort *nc, Reading *oldr, bool is_sub = false) {
- Reading *nr = new Reading(nc);
+ Reading *nr = alloc_reading(nc);
GrammarApplicator *ga = nc->parent->parent->parent;
insert_if_exists(nr->parent->possible_sets, ga->grammar->sets_any);
ga->addTagToReading(*nr, nc->wordform);
@@ -219,7 +219,7 @@ inline Reading *_reading_copy(Cohort *nc, Reading *oldr, bool is_sub = false) {
}
inline Cohort *_cohort_copy(SingleWindow *ns, Cohort *oc) {
- Cohort *nc = new Cohort(ns);
+ Cohort *nc = alloc_cohort(ns);
nc->wordform = _tag_copy(ns->parent->parent, oc->wordform);
boost_foreach (Reading *r, oc->readings) {
Reading *nr = _reading_copy(nc, r);
@@ -275,7 +275,7 @@ void cg3_sentence_addcohort(cg3_sentence *sentence_, cg3_cohort *cohort_) {
cg3_cohort *cg3_cohort_create(cg3_sentence *sentence_) {
SingleWindow *sentence = static_cast<SingleWindow*>(sentence_);
- Cohort *cohort = new Cohort(sentence);
+ Cohort *cohort = alloc_cohort(sentence);
cohort->global_number = sentence->parent->cohort_counter++;
return cohort;
}
@@ -365,7 +365,7 @@ void cg3_cohort_free(cg3_cohort *cohort_) {
cg3_reading *cg3_reading_create(cg3_cohort *cohort_) {
Cohort *cohort = static_cast<Cohort*>(cohort_);
GrammarApplicator *ga = cohort->parent->parent->parent;
- Reading *reading = new Reading(cohort);
+ Reading *reading = alloc_reading(cohort);
insert_if_exists(reading->parent->possible_sets, ga->grammar->sets_any);
ga->addTagToReading(*reading, cohort->wordform);
return reading;
diff --git a/src/version.hpp b/src/version.hpp
index 05bb28e..107b674 100644
--- a/src/version.hpp
+++ b/src/version.hpp
@@ -30,7 +30,7 @@ const char* const CG3_COPYRIGHT_STRING = "Copyright (C) 2007-2015 GrammarSoft Ap
const uint32_t CG3_VERSION_MAJOR = 0;
const uint32_t CG3_VERSION_MINOR = 9;
const uint32_t CG3_VERSION_PATCH = 9;
-const uint32_t CG3_REVISION = 10793;
+const uint32_t CG3_REVISION = 10800;
const uint32_t CG3_FEATURE_REV = 10575;
const uint32_t CG3_TOO_OLD = 10373;
const uint32_t CG3_EXTERNAL_PROTOCOL = 7226;
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/cg3.git
More information about the debian-science-commits
mailing list